Debian Codegraph @

scripts/kbcompletion/data.sh

raw
1 #!/usr/bin/env bash
2 #
3 # Copyright (c) 2017-present, Facebook, Inc.
4 # All rights reserved.
5 #
6 # This source code is licensed under the MIT license found in the
7 # LICENSE file in the root directory of this source tree.
8 #
9 set -e
10 DATADIR=data/
11
12 if [ ! -d "$DATADIR" ]; then
13 mkdir $DATADIR
14 fi
15
16 cd $DATADIR
17 echo "preparing WN18"
18 #wget -P . https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:wordnet-mlj12.tar.gz
19 #mv fetch.php\?media\=en\:wordnet-mlj12.tar.gz wordnet-mlj12.tar.gz
20 wget -P . https://github.com/mana-ysh/knowledge-graph-embeddings/raw/master/dat/wordnet-mlj12.tar.gz
21 tar -xzvf wordnet-mlj12.tar.gz
22 DIR=wordnet-mlj12
23 for f in ${DIR}/wordnet-ml*.txt;
24 do
25 fn=${DIR}/ft_$(basename $f)
26 awk '{print "__label__"$1,"0_"$2, $3;print $1,"1_"$2," __label__"$3}' < ${f} > ${fn};
27 done
28 cat ${DIR}/ft_* > ${DIR}/ft_wordnet-mlj12-full.txt
29 cat ${DIR}/ft_*train.txt ${DIR}/ft_*valid.txt > ${DIR}/ft_wordnet-mlj12-valid+train.txt
30
31 echo "preparing FB15K"
32 #wget https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:fb15k.tgz
33 #mv fetch.php\?media\=en\:fb15k.tgz fb15k.tgz
34 wget https://github.com/mana-ysh/knowledge-graph-embeddings/raw/master/dat/fb15k.tgz
35 tar -xzvf fb15k.tgz
36 DIR=FB15k/
37 for f in ${DIR}/freebase*.txt;
38 do
39 fn=${DIR}/ft_$(basename $f)
40 echo $f " --> " $fn
41 awk '{print "__label__"$1,"0_"$2, $3;print $1,"1_"$2," __label__"$3}' < ${f} > ${fn};
42 done
43 cat ${DIR}/ft_* > ${DIR}/ft_freebase_mtr100_mte100-full.txt
44 cat ${DIR}/ft_*train.txt ${DIR}/ft_*valid.txt > ${DIR}/ft_freebase_mtr100_mte100-valid+train.txt
45
46 echo "preparing FB15K-237"
47 wget https://download.microsoft.com/download/8/7/0/8700516A-AB3D-4850-B4BB-805C515AECE1/FB15K-237.2.zip
48 unzip FB15K-237.2.zip
49 DIR=Release/
50 for f in train.txt test.txt valid.txt
51 do
52 fn=${DIR}/ft_$(basename $f)
53 echo $f " --> " $fn
54 awk -F "\t" '{print "__label__"$1,"0_"$2, $3;print $1,"1_"$2," __label__"$3}' < ${DIR}/${f} > ${fn};
55 done
56 cat ${DIR}/ft_*.txt > ${DIR}/ft_full.txt
57 cat ${DIR}/ft_train.txt ${DIR}/ft_valid.txt > ${DIR}/ft_valid+train.txt
58
59 echo "preparing SVO"
60 wget . https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:svo-tensor-dataset.tar.gz
61 mv fetch.php?media=en:svo-tensor-dataset.tar.gz svo-tensor-dataset.tar.gz
62 tar -xzvf svo-tensor-dataset.tar.gz
63 DIR=SVO-tensor-dataset
64 for f in ${DIR}/svo_data*.dat;
65 do
66 fn=${DIR}/ft_$(basename $f)
67 awk '{print "0_"$1,"1_"$3,"__label__"$2;}' < ${f} > ${fn};
68 done
69 cat ${DIR}/ft_*train*.dat ${DIR}/ft_*valid*.dat > ${DIR}/ft_svo_data-valid+train.dat