BIRL/circle.yml at master · phiwei/BIRL · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
version: 2.0

references:

  install_deps: &install_deps
    run:
      name: Install Packages
      command: |
        sudo apt-get update -qq
        # sudo apt-get install git-lfs
        sudo apt-get install tk-dev pkg-config python-dev python-tk tree
        sudo apt-get install openslide-tools

  install_pips: &install_pips
    run:
      name: Install PyPI dependences
      command: |
        sudo pip install -U backports.functools_lru_cache  # required for matplotlib @py2
        pip install -r requirements.txt --user
        sudo pip install -r ./tests/requirements.txt
        python --version ; pwd ; ls -l
        pip --version ; pip list

  test_coverage: &test_coverage
    run:
      name: Testing and Formating
      command: |
        unset DISPLAY
        mkdir test-reports && mkdir results
        coverage run --source birl -m py.test birl tests bm_dataset bm_experiments bm_ANHIR -v --doctest-modules --junitxml=test-reports/pytest_junit.xml
        flake8 . --max-line-length=100
        codecov
        rm -rf results

  make_docs: &make_docs
   run:
     name: Make Documentation
     command: |
       sudo apt-get install pandoc
       sudo pip install -r docs/requirements.txt
       # sphinx-apidoc -o ./docs/source ./imsegm **/test_* --force --follow-links
       cd docs; make html

  run_dataset: &run_dataset
    run:
      name: Dataset
      command: |
        mkdir output
        python bm_dataset/rescale_tissue_images.py -i "./data_images/rat-kidney_/scale-5pc/*.jpg" --scales 5 -ext .png --nb_workers 2
        python bm_dataset/rescale_tissue_landmarks.py -a ./data_images -d ./output --nb_selected 0.5 --nb_total 200
        python bm_dataset/generate_regist_pairs.py -i "./data_images/images/artificial_*.jpg" -l "./data_images/landmarks/artificial_*.csv" -csv ./data_images/cover_artificial.csv --mode each2all
        # python bm_experiments/create_real_synth_dataset.py -i ./data_images/images/Rat_Kidney_HE.jpg -l ./data_images/landmarks/Rat_Kidney_HE.csv -o ./data_images/synth_dataset -nb 3 --nb_workers 1
        # python bm_experiments/generate_regist_pairs.py -i "./data_images/synth_dataset/*.jpg" -l "./data_images/synth_dataset/*.csv" -csv ./data_images/cover_synth-dataset.csv --mode each2all

  run_experiment: &run_experiment
    run:
      name: General experiments
      command: |
        mkdir applications && mkdir results && touch configs/sample_config.yaml
        # python bm_experiments/bm_comp_perform.py  -o ./results -n 1 # reach time-out while running in 36 threads
        python birl/bm_template.py -t ./data_images/pairs-imgs-lnds_mix.csv -o ./results --visual --unique -cfg configs/sample_config.yaml
        rm ./data_images/*_/*/*_HE.csv  # remove target landmarks from histol. tissue
        python birl/bm_template.py -t ./data_images/pairs-imgs-lnds_histol.csv -d ./data_images -o ./results --preprocessing matching-rgb gray -cfg configs/sample_config.yaml
        python bm_experiments/evaluate_experiment.py -d ./data_images -e ./results/BmTemplate --visual
        tree ./results/BmTemplate

  run_bUnwarpJ: &run_bUnwarpJ
    run:
      name: Experiment - bUnwarpJ
      command: |
        wget https://downloads.imagej.net/fiji/latest/fiji-linux64.zip -O applications/fiji-linux64.zip --progress=bar:force:noscroll
        unzip -q applications/fiji-linux64.zip -d applications/
        rm applications/fiji-linux64.zip
        rm -rf results && mkdir results
        python bm_experiments/bm_bUnwarpJ.py -t ./data_images/pairs-imgs-lnds_mix.csv -o ./results -Fiji ./applications/Fiji.app/ImageJ-linux64 -cfg ./configs/ImageJ_bUnwarpJ_histol.yaml --preprocessing matching-rgb --visual
        python bm_experiments/bm_bUnwarpJ.py -t ./data_images/pairs-imgs-lnds_histol.csv -d ./data_images -o ./results -Fiji ./applications/Fiji.app/ImageJ-linux64 -cfg ./configs/ImageJ_bUnwarpJ-SIFT_histol.yaml --unique
        tree -L 3 ./results
        rm -rf ./applications/Fiji.app

  run_RVSS: &run_RVSS
    run:
      name: Experiment - RVSS
      command: |
        wget https://downloads.imagej.net/fiji/latest/fiji-linux64.zip -O applications/fiji-linux64.zip --progress=bar:force:noscroll
        unzip -q applications/fiji-linux64.zip -d applications/
        rm applications/fiji-linux64.zip
        rm -rf results && mkdir results
        python bm_experiments/bm_RVSS.py -t ./data_images/pairs-imgs-lnds_mix.csv -o ./results -Fiji ./applications/Fiji.app/ImageJ-linux64 -cfg ./configs/ImageJ_RVSS_histol.yaml --visual
        tree -L 3 ./results
        rm -rf ./applications/Fiji.app


jobs:

  Py3-Tests:
    docker:
      - image: circleci/python:3.6
    steps: &steps_tests
      - checkout
      # INSTALLATION
      - *install_deps
      #- run: git lfs pull
      - *install_pips
      # TESTING
      - *test_coverage
      # DOCUMENTATION
      - *make_docs

      # PASSING
      - run:
          name: Finalise
          command: |
            python setup.py install --user
            coverage report && coverage xml -o test-reports/coverage.xml
      # RESULTS
      - store_test_results:
          path: test-reports
      - store_artifacts:
          path: test-reports

  Py3-Experiments:
    docker:
      - image: circleci/python:3.6
    steps: &steps_expts
      - checkout
      # INSTALLATION
      - *install_deps
      #- run: git lfs pull
      - *install_pips
      # EXPERIMENTS
      - *run_dataset
      - *run_experiment
      - *run_bUnwarpJ
      - *run_RVSS

  Py2-Tests:
    docker:
      - image: circleci/python:2.7
    steps: *steps_tests

  Py2-Experiments:
    docker:
      - image: circleci/python:2.7
    steps: *steps_expts


workflows:
  version: 2
  build:
    jobs:
      - Py2-Tests
      - Py3-Tests
      - Py2-Experiments
      - Py3-Experiments