Skip to content

Commit a930c35

Browse files
CharlesGaydonCharlesGaydon
authored
Merge pull request #14 from IGNF/dev
Tests & Refactor (incl. dependencies, CICD workflow, Documentation workflow) & Doc.
2 parents b7ff377 + 269c0ab commit a930c35

File tree

89 files changed

+1708
-1044
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+1708
-1044
lines changed

‎.env_example‎

Lines changed: 1 addition & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
## LOG PATH
22
LOGS_DIR="path/to/dir/"
33

4-
# Prepared data: splitted LAS files in LAS_NAME subdirs in train/val/test subdirs :
5-
# - /PREPARED_DATA_DIR/val/name/000.data
6-
# - /PREPARED_DATA_DIR/val/name/001.data
7-
# ...
4+
## DATA
85
PREPARED_DATA_DIR="path/to/prepared/data/dir"
96

107
## LOGGER

‎.github/workflows/cicd.yaml‎

Lines changed: 80 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,52 +1,99 @@
11
name: CICD
22

3-
# on:
4-
# pull_request:
5-
# branches: [main]
6-
7-
on: push
3+
on:
4+
# Run CICD for non-draft pull request
5+
pull_request:
6+
branches:
7+
- dev
8+
- main
9+
# Also run when the pull request merges (which generates a push)
10+
# So that we can tag the docker image appropriately.
11+
push:
12+
branches:
13+
- dev
14+
- prod
15+
- main
816

917
jobs:
10-
lidar_on_docker:
18+
CICD:
1119
runs-on: self-hosted
1220

1321
steps:
1422

15-
- name: Checkout branch
23+
- name: Checkout branch
1624
uses: actions/checkout@v2
1725

18-
- name: build docker image
19-
run: docker build -t lidar_deep_im .
26+
- name: Build docker image
27+
run: docker build -t myria3d .
2028

2129
- name: Check code neatness (linter)
22-
run: docker run lidar_deep_im flake8
30+
run: docker run myria3d flake8
2331

24-
# - name: unit testing
25-
# run: docker run lidar_deep_im pytest --ignore=actions-runner --ignore="notebooks"
32+
- name: Fast unit tests first
33+
run: >
34+
docker run
35+
-v /var/data/CICD_github_assets/myria3d_B2V0.5/inputs/:/myria3d/tests/files/large/
36+
--ipc=host
37+
myria3d
38+
python -m
39+
pytest -rA -v -m "not slow"
40+
--ignore=actions-runner
2641
2742
- name: Full module run on LAS subset
28-
run: docker run -v /var/data/CICD_github_assets:/CICD_github_assets lidar_deep_im
29-
# sudo mount -v -t cifs -o user=mdaab,domain=IGN,uid=24213,gid=10550 //store.ign.fr/store-lidarhd/projet-LHD/IA/Validation_Module/CICD_github_assets/B2V0.5 /var/data/CICD_github_assets
30-
31-
# - name: Evaluate decisions using optimization code on a single, corrected LAS
32-
# run: >
33-
# docker run -v /var/data/cicd/CICD_github_assets:/CICD_github_assets lidar_deep_im
34-
# python lidar_prod/run.py print_config=true +task='optimize'
35-
# +building_validation.optimization.debug=true
36-
# building_validation.optimization.todo='prepare+evaluate+update'
37-
# building_validation.optimization.paths.input_las_dir=/CICD_github_assets/M8.0/20220204_building_val_V0.0_model/20211001_buiding_val_val/
38-
# building_validation.optimization.paths.results_output_dir=/CICD_github_assets/opti/
39-
# building_validation.optimization.paths.building_validation_thresholds_pickle=/CICD_github_assets/M8.3B2V0.0/optimized_thresholds.pickle
40-
41-
- name: chekc the user
42-
run: whoami
43-
44-
- name: save the docker image because everything worked
45-
run: docker save lidar_deep_im > /var/data/CICD_github_assets/lidar_deep_im.tar # need writing right
46-
47-
- name: clean the server for further uses
43+
run: >
44+
docker run
45+
-v /var/data/cicd/CICD_github_assets/myria3d_B2V0.5/inputs/:/inputs/
46+
-v /var/data/cicd/CICD_github_assets/myria3d_B2V0.5/outputs/:/outputs/
47+
--ipc=host
48+
myria3d
49+
python -m
50+
myria3d.predict
51+
--config-path /inputs/
52+
--config-name predict_config_V2.0.0.yaml
53+
predict.src_las=/inputs/792000_6272000_subset_buildings.las
54+
predict.output_dir=/outputs/
55+
predict.ckpt_path=/inputs/RandLaNet_Buildings_B2V0.5_epoch_033.ckpt
56+
datamodule.batch_size=25
57+
hydra.run.dir=/myria3d
58+
59+
# Always run with --ipc=host to enable sufficient shared memory when predicting on large data
60+
- name: Slow unit tests last, using large files stored in self-hosted action-runner.
61+
run: >
62+
docker run
63+
-v /var/data/cicd/CICD_github_assets/myria3d_B2V0.5/inputs/:/myria3d/tests/data/large/
64+
--ipc=host
65+
myria3d
66+
python -m
67+
pytest -rA -v -m "slow"
68+
--ignore=actions-runner
69+
70+
# Everything ran so we tag the valid docker image to keep it
71+
# This happens for push events, which are in particular
72+
# triggered when a pull request is merged.
73+
- name: Get the branch name into an environment variable.
74+
if: github.event_name == 'push'
75+
uses: nelonoel/branch-name@v1.0.1
76+
77+
- name: Print the branch name
78+
if: github.event_name == 'push'
79+
run: echo ${BRANCH_NAME}
80+
81+
- name: Tag the docker image with branch name
82+
if: github.event_name == 'push'
83+
run: docker tag myria3d:latest myria3d:${BRANCH_NAME}
84+
85+
- name: Run the new, tagged image at least once so that is it not prunned by mistake when using docker system prune
86+
if: github.event_name == 'push'
87+
run: docker run myria3d:${BRANCH_NAME} bash
88+
89+
# This needs writing rights to the mounted path
90+
- name: Save the docker image as myria3d_${BRANCH_NAME}.tar
91+
if: github.event_name == 'push'
92+
run: docker save myria3d:${BRANCH_NAME} -o /var/data/cicd/CICD_github_assets/CICD_docker_images/myria3d_${BRANCH_NAME}.tar
93+
94+
- name: Clean dangling docker images
4895
if: always() # always do it, even if something failed
49-
run: docker system prune # remove obsolete docker images (take a HUGE amount of space)
96+
run: docker system prune --force # remove dangling docker images, without asking user for confirmation
5097

5198

5299

‎.github/workflows/gh-pages‎

Lines changed: 0 additions & 52 deletions
This file was deleted.

‎.github/workflows/gh-pages.yml‎

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,15 @@ on:
66
push: # <- trigger when we call push
77
branches:
88
- main # <- but only on main branch
9-
- FixDocAPI # <- also on this branch until documentation is up and running.
109

1110
jobs:
1211

1312
build-and-deploy:
1413
runs-on: ubuntu-latest
15-
14+
# Use bash instead of sh for conda activation
15+
defaults:
16+
run:
17+
shell: bash -l {0}
1618
# Task to do when launching the workflow
1719
steps:
1820

@@ -22,29 +24,49 @@ jobs:
2224
uses: actions/checkout@v2
2325

2426
# 2. Sphinx part : install tool and dependencies
27+
# See https://github.com/conda-incubator/setup-miniconda#caching-environments
2528

26-
- name: "Set up Python"
27-
uses: actions/setup-python@v1
29+
- name: Setup a conda-incubator with an empty conda env
30+
uses: conda-incubator/setup-miniconda@v2
2831
with:
29-
python-version: 3.9.12
32+
python-version: 3.9.12
33+
miniforge-variant: Mambaforge
34+
miniforge-version: latest
35+
use-mamba: true
36+
# For caching to work:
37+
use-only-tar-bz2: true
38+
# Environment to create and activate for next steps
39+
activate-environment: myria3d
3040

31-
# Packages that depend on torch need need to be installed afterwards,
32-
# hence the "requirements_torch_deps.txt file.
33-
- name: "Install Python dependencies"
34-
working-directory: ./docs/
35-
run: |
36-
python3 -m pip install --upgrade pip
37-
pip3 install -r requirements.txt
38-
pip3 install -r requirements_torch_deps.txt
41+
# Cache the conda environment
42+
# See https://github.com/conda-incubator/setup-miniconda#caching-environments
43+
- name: Get Date
44+
id: get-date
45+
run: echo "::set-output name=today::$(/bin/date -u '+%Y%m%d')"
46+
shell: bash
47+
48+
- name: Cache conda environment
49+
uses: actions/cache@v2
50+
with:
51+
path: ${{ env.CONDA }}/envs
52+
key: conda-${{ runner.os }}--${{ runner.arch }}--${{ steps.get-date.outputs.today }}-${{ hashFiles('setup_env/requirements.yml') }}-${{ hashFiles('setup_env/requirements.txt') }}
53+
env:
54+
# Increase this value to reset cache if etc/example-environment.yml has not changed
55+
CACHE_NUMBER: 0
56+
id: cache
57+
58+
- name: Update environment if there was no cached env.
59+
run: mamba env update -n myria3d -f setup_env/requirements.yml
60+
if: steps.cache.outputs.cache-hit != 'true'
3961

62+
- name: List installed packages
63+
run: mamba list
4064

4165
- name: "Build Sphinx Doc"
4266
working-directory: ./docs/
43-
run: |
44-
make html
67+
run: make html
4568

4669
# 3. Déploiement sur les Github Pages
47-
4870
- name: "Deploy Github Pages"
4971
uses: JamesIves/github-pages-deploy-action@3.7.1
5072
with:

‎.gitignore‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
1-
# Usr defined
1+
# User defined
22
nohup.out
3+
inputs/
34
outputs/
4-
bash/schedule/*
5+
checkpoints/
6+
runs/
7+
# large files that are not tracked
8+
tests/data/large
9+
510

611
# Byte-compiled / optimized / DLL files
712
__pycache__/

‎Dockerfile‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
FROM nvidia/cuda:10.2-devel-ubuntu18.04
2+
# An nvidia image seems to be necessary for torch-points-kernel.
3+
# Also, a "devel" image seems required for the same library
4+
5+
# set the IGN proxy, otherwise apt-get and other applications don't work
6+
# Should be commented out outside of IGN
7+
ENV http_proxy 'http://192.168.4.9:3128/'
8+
ENV https_proxy 'http://192.168.4.9:3128/'
9+
10+
# set the timezone, otherwise it asks for it... and freezes
11+
ENV TZ=Europe/Paris
12+
RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
13+
14+
# Needed to use apt-get afterwards due to CUDA changes described here since April 27, 2022:
15+
# https://forums.developer.nvidia.com/t/notice-cuda-linux-repository-key-rotation/212772
16+
# Not the recommpended method, but else we need wget installed afterwards.
17+
# We changed to 10.2-devel-ubuntu18.04 so that might not be needed.
18+
RUN apt-get update && apt-get upgrade -y && apt-get install -y wget
19+
RUN apt-key del 7fa2af80
20+
RUN apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
21+
22+
# all the apt-get installs
23+
RUN apt-get update && apt-get upgrade -y && apt-get install -y \
24+
software-properties-common \
25+
wget \
26+
git \
27+
libgl1-mesa-glx libegl1-mesa libxrandr2 libxrandr2 libxss1 libxcursor1 libxcomposite1 libasound2 libxi6 libxtst6 # package needed for anaconda
28+
29+
RUN wget --quiet https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O ~/miniconda.sh \
30+
&& /bin/bash ~/miniconda.sh -b -p /opt/conda \
31+
&& rm ~/miniconda.sh
32+
33+
ENV PATH /opt/conda/bin:$PATH
34+
35+
# Only copy necessary files to set up the environment,
36+
# to use docker caching if requirements files were not updated.
37+
WORKDIR /setup_env
38+
COPY ./setup_env/ .
39+
40+
# install mamba to setup the env faster
41+
RUN conda install -y mamba -n base -c conda-forge
42+
# Build the environment
43+
RUN mamba env create -f requirements.yml
44+
45+
# Copy the repository content in /myria3d
46+
WORKDIR /myria3d
47+
COPY . .
48+
49+
# Make RUN commands use the new environment:
50+
SHELL ["conda", "run", "-n", "myria3d", "/bin/bash", "-c"]
51+
52+
# the entrypoint garanty that all command will be runned in the conda environment
53+
ENTRYPOINT ["conda", \
54+
"run", \
55+
"-n", \
56+
"myria3d"]
57+
58+
# Example usage
59+
CMD ["python", \
60+
"-m", \
61+
"myria3d.predict", \
62+
"--config-path", \
63+
"/CICD_github_assets/parametres_etape1/.hydra", \
64+
"--config-name", \
65+
"predict_config_V1.6.3.yaml", \
66+
"predict.src_las=/CICD_github_assets/parametres_etape1/test/792000_6272000_subset_buildings.las", \
67+
"predict.output_dir=/CICD_github_assets/output_etape1", \
68+
"predict.ckpt_path=/CICD_github_assets/parametres_etape1/checkpoints/epoch_033.ckpt", \
69+
"predict.gpus=0", \
70+
"datamodule.batch_size=10", \
71+
"datamodule.subtile_overlap=0", \
72+
"hydra.run.dir=/myria3d"]
73+

0 commit comments

Comments
 (0)