netJina/fetch_data at master · nworbde/netJina · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
#!/bin/bash

function check_okay {
    if [ $? -ne 0 ]; then
        echo
        pwd
        echo "FAILED"
        echo
        exit 1
    fi
}

function fetch_and_verify {
    # if file exists and is verified, then no need to fetch from repository
    need_to_fetch=1
    lib=$1
    chks=$2
    url=$3
    if [ -f $lib ]; then
        echo "$lib exists...verifying"
        verify_datafile $lib $chks
        if [ $? -ne 0 ]; then
            echo "verification failed"
            need_to_fetch=1
        else
            echo "verification okay"
            need_to_fetch=0
        fi
    fi

    if [ $need_to_fetch -eq 1 ]; then
        echo "fetching $lib from $url..."
        get_datafile $lib $url
        echo "verifying $lib..."
        verify_datafile $lib $chks
        check_okay
    fi
}

function get_datafile {
    zip='.gz'
    db=$1
    tmpfile="tmp_$db"
    url=$2
    unzip=`which gunzip`
    curl -fS $url | $unzip > $tmpfile
    check_okay
    mv -i $tmpfile $db
    check_okay
}

function verify_datafile {
    db=$1
    key_cksm=$2
    chk=""
    if [ `which md5sum` ]; then
        chk=`which md5sum`
        this_cksm=`$chk $db`
    elif [ `which md5` ]; then
        chk=`which md5`
        this_cksm=`$chk -q $db`
    else
        echo "unable to find md5 or md5sum utility...skipping verification"
        return 0
    fi
    echo "checksum is  $this_cksm"
    echo "compare with $key_cksm"
    if [ $this_cksm !=  $key_cksm ]; then
        echo "FAILED"
        return 1
    fi
    return 0
}

#databases
RLIB=reaclib_db
NLIB=nuclib_db
SLIB=starlib_db
SPAR=starlib_pfcn_db

#checksums
MD5_NLIB=2621b33271bec07f306922c8d72bce88
MD5_RLIB=06acba5a45a0f343a3a5f98cb522bbea
MD5_SLIB=2c6d666e395af3223e50da5a9566d09f
MD5_SPAR=743e6599dbe0aac097c61ad8a99f9259

#locations
RLIB_URL=https://dl.dropboxusercontent.com/u/52649885/netJina/20140508default2.gz
NLIB_URL=https://dl.dropboxusercontent.com/u/52649885/netJina/winvne_v2.0.dat.gz
SLIB_URL=https://dl.dropboxusercontent.com/u/52649885/netJina/starlib_v4.dat.gz
SPAR_URL=https://dl.dropboxusercontent.com/u/52649885/netJina/partition.dat.gz

echo
echo "Installing Reaclib, Nuclib, and Starlib Datafiles"
echo "-------------------------------------------------"

echo "establishing data directory..."
mkdir -p data
check_okay
cd data
mkdir -p cache
check_okay

fetch_and_verify $NLIB $MD5_NLIB $NLIB_URL
check_okay
fetch_and_verify $RLIB $MD5_RLIB $RLIB_URL
check_okay
fetch_and_verify $SLIB $MD5_SLIB $SLIB_URL
check_okay
fetch_and_verify $SPAR $MD5_SPAR $SPAR_URL
check_okay

echo "done"
echo

cd ..