VERSION=0.24 BIN_OPT:= \ create-graph.native \ print-stats.native \ bin2src.native \ src2bin.native \ build-fixpoint.native \ clean-repository.native \ buildgraph2srcgraph.native \ annotate-strong.native \ partial-order.native \ find-fvs.native \ collapse-srcgraph.native \ optuniv.native \ calculate-fas.native \ buildcheck-more-problems.native \ distcheck-more-problems.native BIN_BYTE:=$(patsubst %.native, %.byte, $(BIN_OPT)) BIN_DEBUG:=$(patsubst %.native, %.d.byte, $(BIN_OPT)) BIN_PROF:=$(patsubst %.native, %.p.native, $(BIN_OPT)) BIN_PYTOOLS:=$(patsubst tools/%.py, %, $(filter-out tools/util.py tools/debarch.py,$(wildcard tools/*.py))) BIN_OCTOOLS:=$(patsubst %.native, %, $(BIN_OPT)) BIN_SHTOOLS:=$(patsubst tools/%.sh, %, $(wildcard tools/*.sh)) BIN_TOOLS:=$(BIN_PYTOOLS) $(BIN_OCTOOLS) $(BIN_SHTOOLS) MANPAGES:=$(patsubst %, doc/man/botch-%.1, $(BIN_TOOLS)) OCAMLBUILD_BEST ?= $(if $(wildcard /usr/bin/ocamlopt),native,byte) PWD := $(shell pwd) BUILD = $(PWD)/dose/_build DOSELIBS = $(PWD)/dose/_build/dose3 CUDFLIBS = $(PWD)/dose/_build/cudf DOSEBYTELIBS = \ _build/doselibs/common.cma \ _build/doselibs/debian.cma \ _build/doselibs/versioning.cma \ _build/doselibs/csw.cma \ _build/doselibs/pef.cma \ _build/doselibs/algo.cma \ _build/doselibs/doseparse.cma \ _build/doselibs/doseparseNoRpm.cma # we turn on all warnings but turn off: # 4 Fragile pattern matching: matching that will remain complete even if additional constructors are added to one of the variant types matched. # 9 Missing fields in a record pattern. # 24 Bad module name: the source file name is not a valid OCaml module name. CFLAGS=-cflags "-w +a-4-9-24" bindir=/usr/bin datadir=/usr/share/botch mandir=/usr/share/man # we have to export PYTHONHASHSEED because otherwise networkx will create # content with random output order # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=749710 # https://github.com/networkx/networkx/issues/1181 # on the other hand, the generated hashes are not the same on 32 vs 64 arches # http://bugs.python.org/issue22621 # http://bugs.debian.org/778373 export PYTHONHASHSEED=0 # to make the sort order in the tests locale independent export LC_COLLATE=C.UTF-8 export LC_ALL=C.UTF-8 # because dose2html outputs utf8 on standard output export PYTHONIOENCODING=utf-8 .PHONY: all all: $(OCAMLBUILD_BEST) wiki man .PHONY: native native: atdgen OCAMLPATH=$(BUILD) ocamlbuild -lflags -runtime-variant,_pic -classic-display -use-ocamlfind $(CFLAGS) $(BIN_OPT) .PHONY: byte byte: atdgen OCAMLPATH=$(BUILD) ocamlbuild -classic-display -use-ocamlfind $(CFLAGS) $(BIN_BYTE) .PHONY: debug debug: atdgen OCAMLPATH=$(BUILD) ocamlbuild -classic-display -use-ocamlfind $(CFLAGS) $(BIN_DEBUG) # profiling for use of native executables with gprof .PHONY: profile profile: atdgen OCAMLPATH=$(BUILD) ocamlbuild -classic-display -use-ocamlfind $(CFLAGS) $(BIN_PROF) doc/man/botch.1: $(MANPAGES) ./doc/man/generate-botch-manpage.py | pod2man --section 1 --center="botch tools" --name botch > $@ .PHONY: man man: $(MANPAGES) doc/man/botch.1 .PHONY: wiki wiki: $(MAKE) -C doc/wiki doc/man/%.1: doc/man/%.pod pod2man --section 1 --center="botch tools" $< > $@ .PHONY: doselib doselib: (cd dose && ./configure --with-zip --with-bz2 && make clean libs $(DOSEBYTELIBS)) rm -Rf $(DOSELIBS) mkdir -p $(DOSELIBS) cp dose/META $(DOSELIBS) for i in \ dose/_build/*/common.* \ dose/_build/*/versioning.* \ dose/_build/*/debian.* \ dose/_build/*/algo.* \ dose/_build/*/csw.* \ dose/_build/*/pef.* \ dose/_build/*/doseparseNoRpm.* \ dose/_build/*/doseparse.*; do \ if [ -e $$i ]; then \ cp $$i $(DOSELIBS) ; \ rm -f $(DOSELIBS)/*.mlpack $(DOSELIBS)/*.cmx ; \ fi ; \ done rm -Rf $(CUDFLIBS) mkdir -p $(CUDFLIBS) cp dose/cudf/META $(CUDFLIBS) cp dose/_build/doselibs/cudf*.cmi $(CUDFLIBS) for i in dose/_build/doselibs/cudf.*; do \ if [ -e $$i ]; then \ cp $$i $(CUDFLIBS) ; \ rm -f $(CUDFLIBS)/*.mlpack $(CUDFLIBS)/*.cmx ; \ fi ; \ done %_j.ml: %.atd atdgen -j -j-std $< %_t.ml: %.atd atdgen -t $< .PHONY: atdgen atdgen: datatypes_j.ml datatypes_t.ml # the "while : ; do" loop around graph-difference is a temporary fix to work # around random segmentation faults. See http://bugs.python.org/issue24605 define diff_tmp_out for t in tmp out; do \ for f in tests/$(1)/$$t/* $$t/*; do basename "$$f"; done | sort | uniq | while read f; do \ echo checking $$f; \ case "$$f" in \ *.xml|*.dot) \ echo "+ ./tools/graph-difference.py \"tests/$(1)/$$t/$$f\" \"$$t/$$f\""; \ while : ; do ./tools/graph-difference.py "tests/$(1)/$$t/$$f" "$$t/$$f"; exit=$$?; if [ $$exit -eq 139 ]; then echo segfault; continue; fi; if [ $$exit -ne 0 ]; then exit 1; else break; fi; done;; \ *) \ echo "+ diff -u \"tests/$(1)/$$t/$$f\" \"$$t/$$f\""; \ cmp --silent "tests/$(1)/$$t/$$f" "$$t/$$f" || { diff -u "tests/$(1)/$$t/$$f" "$$t/$$f" | head --lines 100 && exit 1; };; \ esac; \ done; \ done endef .PHONY: test-misc test-misc: $(OCAMLBUILD_BEST) rm -rf tmp out mkdir -p tmp out $(eval packages := tests/sid-amd64-packages-20160830T000000Z) $(eval sources := tests/sid-sources-20160830T000000Z) grep-dctrl --exact-match --field Package build-essential $(packages) \ | ./tools/latest-version.py - - > tmp/build-essential ./bin2src.$(OCAMLBUILD_BEST) --deb-native-arch=amd64 tmp/build-essential $(sources) \ > tmp/build-essential-src ./create-graph.$(OCAMLBUILD_BEST) --deb-native-arch=amd64 --bg $(sources) $(packages) tmp/build-essential-src \ > tmp/selfcontained_repo.xml ./tools/buildgraph2packages.py tmp/selfcontained_repo.xml $(packages) \ > tmp/packages ./bin2src.$(OCAMLBUILD_BEST) --deb-native-arch=amd64 tmp/packages $(sources) \ > tmp/sources # test botch-calcportsmetric ./create-graph.$(OCAMLBUILD_BEST) --deb-drop-b-d-indep --deb-native-arch=amd64 tmp/packages tmp/sources --strongtype > tmp/strongbuildgraph.xml ./buildgraph2srcgraph.$(OCAMLBUILD_BEST) tmp/strongbuildgraph.xml --deb-drop-b-d-indep --deb-native-arch=amd64 tmp/packages tmp/sources > tmp/strongsrcgraph.xml ./create-graph.$(OCAMLBUILD_BEST) --deb-drop-b-d-indep --deb-native-arch=amd64 tmp/packages tmp/sources --closuretype > tmp/closurebuildgraph.xml ./buildgraph2srcgraph.$(OCAMLBUILD_BEST) tmp/closurebuildgraph.xml --deb-drop-b-d-indep --deb-native-arch=amd64 tmp/packages tmp/sources > tmp/closuresrcgraph.xml ./tools/calcportsmetric.py tmp/strongsrcgraph.xml tmp/closuresrcgraph.xml > out/importance_metric.txt # test botch-multiarch-interpreter-problem dose-ceve --deb-drop-b-d-indep --deb-native-arch=amd64 -G pkg -T grml deb://tmp/packages debsrc://tmp/sources > tmp/ma_interpreter.xml ./tools/multiarch-interpreter-problem.py --packages tmp/packages tmp/ma_interpreter.xml > out/ma_interpreter.txt # test y-u-b-d-transitive-essential ./tools/y-u-b-d-transitive-essential.sh --debug --verbose --develop --tmp tmp --output out amd64 tmp/packages tmp/sources acl # remove svg because graphviz changes its output format slightly every other version rm out/acl_2.2.52-3_n20.svg # test botch-buildcheck-more-problems ./buildcheck-more-problems.$(OCAMLBUILD_BEST) --verbose --progress --checkonly=plasma-desktop,haskell-hledger-ui --deb-native-arch=amd64 --explain --failures $(packages) $(sources) > out/buildcheck.yaml || [ $$? -eq 1 ] # test botch-buildcheck-more-problems cross because we want to be able # to check yaml output that contains a depchain with an implicit # dependency on a package that is Essential:yes (and thus there is no # "depends" field) ./tools/convert-arch.py amd64 armhf tmp/packages - | grep-dctrl -X \( --not -FArchitecture all --and --not -FMulti-Arch foreign \) > tmp/packages_armhf_noall_nomaforeign ./buildcheck-more-problems.$(OCAMLBUILD_BEST) --verbose --progress --checkonly=apache2,ant --deb-native-arch=amd64 --deb-host-arch=armhf --explain --failures tmp/packages_armhf_noall_nomaforeign tests/crossbuild-essential-armhf tmp/packages tmp/sources > out/buildcheck_cross.yaml || [ $$? -eq 1 ] # test botch-distcheck-more-problems ./distcheck-more-problems.$(OCAMLBUILD_BEST) --verbose --progress --checkonly=task-kde-desktop:amd64,libghc-hledger-dev:amd64 --deb-native-arch=amd64 --explain --failures deb://$(packages) > out/distcheck.yaml || [ $$? -eq 1 ] # test dose2html ./tools/dose2html.py --srcsdir=out --wwwroot=out/ --packages=$(packages) out/distcheck.yaml out/distcheck.html ./tools/dose2html.py --srcsdir=out --wwwroot=out/ --packages=$(packages) out/buildcheck.yaml out/buildcheck.html ./tools/dose2html.py --srcsdir=out --wwwroot=out/ --packages=tmp/packages_armhf_noall_nomaforeign --packages=tests/crossbuild-essential-armhf --packages=tmp/packages out/buildcheck_cross.yaml out/buildcheck_cross.html # test networkx dot read/write because there constantly seem to be # problems with that: # - endless loop for graphs with attributes in pygraphviz # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=794444 # https://github.com/pygraphviz/pygraphviz/issues/65 # - pygraphviz cannot handle graph attributes # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=819739 # - networkx cannot store graph in dot format where a node name contains a colon # https://github.com/networkx/networkx/issues/2050 # - python3 support # https://github.com/erocarrera/pydot/issues/76 # - dead pydot upstream and too many forks # https://github.com/carlos-jenkins/pydotplus/issues/1 # - exclusive use of non-packaged pydotplus fork by networkx # https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=819480 # - undeterministic output order # https://github.com/networkx/networkx/issues/1181 # https://github.com/networkx/networkx/issues/1267 # - unicode encode/decode problems # https://github.com/erocarrera/pydot/issues/24 # - networkx API breakages: removal of nx.write_dot and nx.read_dot # http://networkx.readthedocs.io/en/stable/reference/api_1.11.html # - pygraphviz API breakages: AGraph dropped the file parameter # https://github.com/pygraphviz/pygraphviz/commit/21d9153c25ff5d8047c46f48f043f0788ab10da5 # - pygraphviz segfauls when reading from a io.StringIO # https://github.com/pygraphviz/pygraphviz/issues/101 # - pygraphviz adds the input filename as a graph attribute # https://github.com/pygraphviz/pygraphviz/issues/102 # - networkx doesn't parse dot data from memory # https://github.com/networkx/networkx/issues/2249 dose-ceve --deb-drop-b-d-indep -c "src:acl" --deb-native-arch=amd64 -G pkg -T dot deb://tmp/packages debsrc://tmp/sources > tmp/aclbd.dot ./tools/graph-shortest-path.py tmp/aclbd.dot --all --source __ID__:"src:acl (= 2.2.52-3)" --target __ID__:"multiarch-support:amd64 (= 2.23-5)" > out/acl-ma-path.dot ./tools/graph-ancestors.py --target __ID__:"perl:amd64 (= 5.22.2-3)" tmp/aclbd.dot > out/acl_pred.dot ./tools/graph-descendants.py --source __ID__:"perl:amd64 (= 5.22.2-3)" tmp/aclbd.dot > out/acl_succ.dot ./tools/graph-neighborhood.py --depth=2 --center __ID__:"perl:amd64 (= 5.22.2-3)" tmp/aclbd.dot > out/acl_neighbors.dot # test y-u-no-bootstrap # Remove yui-compressor to make src:doxygen bd-uninstallable grep-dctrl --not --field Package yui-compressor tmp/packages > tmp/packages_noyui ./tools/y-u-no-bootstrap.sh --debug --verbose --develop --tmp tmp --output out amd64 tmp/packages_noyui tmp/sources # remove svg because graphviz changes its output format slightly every other version rm out/doxygen_1.8.11-3.svg # verify results $(call diff_tmp_out,misc) .PHONY: test-default test-default: $(OCAMLBUILD_BEST) rm -rf tmp out ./tools/native.sh --debug --verbose --develop --output out --tmp tmp --deb-drop-b-d-indep amd64 tests/sid-amd64-packages-20160830T000000Z tests/sid-sources-20160830T000000Z $(call diff_tmp_out,default) .PHONY: test-selfcontained test-selfcontained: $(OCAMLBUILD_BEST) rm -rf tmp out ./tools/native.sh --debug --verbose --develop --output out --tmp tmp --deb-drop-b-d-indep --optgraph --latest --clean --self-contained --optuniv --sapsb --strong --no-drop amd64 tests/sid-amd64-packages-20160830T000000Z tests/sid-sources-20160830T000000Z $(call diff_tmp_out,selfcontained) .PHONY: test-cross test-cross: $(OCAMLBUILD_BEST) rm -rf tmp out mkdir out cp tests/cross-ma.diff out/ma.diff ./tools/cross.sh --debug --verbose --develop --output=out --tmp=tmp --deb-drop-b-d-indep --optgraph amd64 armhf tests/sid-amd64-packages-20160830T000000Z tests/sid-sources-20160830T000000Z $(call diff_tmp_out,cross) .PHONY: test-man test-man: $(OCAMLBUILD_BEST) # test whether there is no manpage without a program by # normalizing program and manpage names and only printing # those names which are not duplicate but unique ls tools/*.py *.$(OCAMLBUILD_BEST) tools/*.sh doc/man/botch-*.pod \ | grep -v tools/util.py \ | grep -v tools/debarch.py \ | while read pkg; do \ pkg=`basename $$pkg .$(OCAMLBUILD_BEST)`; \ pkg=`basename $$pkg .py`; \ pkg=`basename $$pkg .sh`; \ pkg=`basename $$pkg .pod`; \ echo $${pkg#botch-}; \ done | sort | uniq -u | while read pkg; do \ echo doc/man/botch-$${pkg}.pod is superfluous >&2; \ exit 1; \ done # check that every pod file includes a synopsis and a description # section and that the synopsis includes the right program name for f in doc/man/*.pod; do \ grep -q '^=head1 SYNOPSIS$$' "$$f"; \ grep -q '^=head1 DESCRIPTION$$' "$$f"; \ name=$$(basename "$$f" .pod); \ ret=0; \ awk '/^=head1 SYNOPSIS$$/{flag=1;next}/^=head1 DESCRIPTION$$/{flag=0}flag' \ "$$f" | grep -q '^=item B<'"$$name"'> ' || ret=1; \ if [ $$ret -ne 0 ]; then \ echo "wrong synopsis in $$f" >&2; \ exit 1; \ fi; \ done # now that we know that there is one man page for every program, check # whether the man page documents all the program options ls tools/*.py *.$(OCAMLBUILD_BEST) tools/*.sh \ | grep -v tools/util.py \ | grep -v tools/debarch.py \ | grep -v buildcheck-more-problems | grep -v distcheck-more-problems \ | while read pkg; do \ shortopts=`./$$pkg --help 2>&1 | sed -n '/^options:$$/,$$p' | sed -ne 's/^ \{1,4\}-\([^ =-]\).*$$/\1/p' | sort`; \ longopts=`./$$pkg --help 2>&1 | sed -n '/^options:$$/,$$p' | sed -ne 's/^ \{1,4\}\(-[^ =-]\+,\)\? --\([^ =]\+\)\([ =].*\)\?$$/\2/p' | sort`; \ echo "compare \`./$$pkg --help\` with man page..."; \ man=`basename $$pkg .$(OCAMLBUILD_BEST)`; \ man=`basename $$man .py`; \ man=`basename $$man .sh`; \ man=`basename $$man .pod`; \ longman=`sed -ne 's/=item B<[^>]*--\([^>=]\+\)=\?>.*$$/\1/p' doc/man/botch-$${man}.pod | sort`; \ shortman=`sed -ne 's/=item B<-\([^>,-]\)[^>]*>.*$$/\1/p' doc/man/botch-$${man}.pod | sort`; \ if [ "$$shortopts" != "$$shortman" ]; then \ echo "\`./$$pkg --help\` short options and man page ./$$man disagree:"; \ echo "$$shortopts != $$shortman"; \ exit 1; \ fi; \ if [ "$$longopts" != "$$longman" ]; then \ echo "\`./$$pkg --help\` long options and man page ./$$man disagree:"; \ echo "$$longopts != $$longman"; \ exit 1; \ fi; \ done .PHONY: test-python test-python: # FIXME: add more tests #./tests.py #OCAMLPATH=$(BUILD) ocamlbuild -classic-display -use-ocamlfind $(CFLAGS) tests.$(OCAMLBUILD_BEST) #./tests.$(OCAMLBUILD_BEST) pyflakes3 tools/*.py # E402 is triggered by the sys.path.append() statement in front of import of utils # see https://github.com/PyCQA/pycodestyle/issues/264 # W503 is not PEP8 compliant # E203 is not PEP8 compliant pycodestyle --max-line-length=88 --ignore=E402,E203,W503 tools/*.py black --check tools/*.py .PHONY: test test: test-python test-man test-default test-selfcontained test-cross test-misc .PHONY: install-man install-man: man mkdir -p $(DESTDIR)$(mandir)/man1/ cp $(MANPAGES) $(DESTDIR)$(mandir)/man1/ cp doc/man/botch.1 $(DESTDIR)$(mandir)/man1/ .PHONY: install-wiki install-wiki: wiki $(MAKE) -C doc/wiki install .PHONY: install-bin install-bin: $(OCAMLBUILD_BEST) mkdir -p $(DESTDIR)$(bindir) for octool in $(BIN_OCTOOLS); do \ cp _build/$$octool.$(OCAMLBUILD_BEST) $(DESTDIR)$(bindir)/botch-$$octool; \ done for shtool in $(BIN_SHTOOLS); do \ cp tools/$$shtool.sh $(DESTDIR)$(bindir)/botch-$$shtool; \ done for pytool in $(BIN_PYTOOLS); do \ cp tools/$$pytool.py $(DESTDIR)$(bindir)/botch-$$pytool; \ done mkdir -p $(DESTDIR)$(datadir) cp -r droppable $(DESTDIR)$(datadir) cp tools/util.py $(DESTDIR)$(datadir) cp tools/debarch.py $(DESTDIR)$(datadir) .PHONY: install install: install-bin install-man install-wiki .PHONY: clean clean: ocamlbuild -clean rm -f datatypes_j.ml datatypes_j.mli datatypes.ml datatypes.mli datatypes_t.ml datatypes_t.mli rm -f tools/*.pyc rm -f *.native *.byte *.p.native *.d.byte rm -f doc/man/*.1 $(MAKE) -C doc/wiki clean .PHONY: distclean distclean: clean rm -rf tools/__pycache__ tmp out .PHONY: doseclean doseclean: (cd dose && ocamlbuild -clean) .PHONY: tarball tarball: $(eval tmpdir := $(shell mktemp --directory)) git archive --worktree-attributes --prefix=botch-$(VERSION)/ -o $(tmpdir)/botch-$(VERSION).tar HEAD git -C doc/wiki archive --worktree-attributes --prefix=botch-$(VERSION)/doc/wiki/ -o $(tmpdir)/botch-doc.tar HEAD git -C tests archive --worktree-attributes --prefix=botch-$(VERSION)/tests/ -o $(tmpdir)/botch-tests.tar HEAD # tar --concatenate seems to only take two files as input so we can # neither replace the temporary files with a pipe nor combine both of # the below commands into one... :( tar --concatenate -f $(tmpdir)/botch-$(VERSION).tar $(tmpdir)/botch-doc.tar tar --concatenate -f $(tmpdir)/botch-$(VERSION).tar $(tmpdir)/botch-tests.tar # testing shows that in this case, -9e compresses better than -9 xz --verbose -c9e $(tmpdir)/botch-$(VERSION).tar > ../botch-$(VERSION).tar.xz rm -rf $(tmpdir) .PHONY: upload upload: tarball gpg -u 8FBD83E1 --armor --detach-sig ../botch-$(VERSION).tar.xz scp ../botch-$(VERSION).tar.xz.asc ../botch-$(VERSION).tar.xz mmfn:/var/www/botch/ ssh mmfn tree -s --noreport -P "botch-*.tar.xz*" -o /var/www/botch/index.html -v -L 1 -H "." -T "botch-releases" /var/www/botch/ # Behave like POSIX. In particular, if this target is mentioned then recipes # will be invoked as if the shell had been passed the -e flag: the first # failing command in a recipe will cause the recipe to fail immediately. .POSIX: