From 5abb7a9eaf55edf8769685b55a7ce5e87d8ff518 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 8 Sep 2022 10:17:18 +0200
Subject: [PATCH 01/20] ADD: GEOS support
---
tools/docker/build-deps-cli.txt | 4 ++++
tools/docker/build-flags-otb.txt | 2 ++
2 files changed, 6 insertions(+)
diff --git a/tools/docker/build-deps-cli.txt b/tools/docker/build-deps-cli.txt
index ffd72911..67d94187 100644
--- a/tools/docker/build-deps-cli.txt
+++ b/tools/docker/build-deps-cli.txt
@@ -44,3 +44,7 @@ libopencv-ml-dev
libsvm-dev
libtinyxml-dev
zlib1g-dev
+libgeos++-dev
+libgeos-3.8.0
+libgeos-c1v5
+libgeos-dev
diff --git a/tools/docker/build-flags-otb.txt b/tools/docker/build-flags-otb.txt
index def7bd2b..fe0eb08d 100644
--- a/tools/docker/build-flags-otb.txt
+++ b/tools/docker/build-flags-otb.txt
@@ -28,3 +28,5 @@
-DOTB_USE_GLUT=OFF
-DOTB_USE_GLEW=OFF
-DOTB_USE_GLFW=OFF
+
+-DGDAL_SB_EXTRA_OPTIONS=--with-geos
--
GitLab
From e8dfec4062f5005e82785a8e786c3fbd36f3849b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 13 Sep 2022 10:24:02 +0200
Subject: [PATCH 02/20] ADD: check existence of output_shapes.json and
output_types.json
---
otbtf/tfrecords.py | 3 +++
1 file changed, 3 insertions(+)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 123bdea5..1acd01d0 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -121,6 +121,9 @@ class TFRecords:
:param preprocessing_fn: Optional. A preprocessing function that process the input example
:param kwargs: some keywords arguments for preprocessing_fn
"""
+ for file in [self.output_types_file, self.output_shapes_file]:
+ assert self.output_types, f"The file {file} is missing!"
+
read_features = {key: tf.io.FixedLenFeature([], dtype=tf.string) for key in self.output_types}
example_parsed = tf.io.parse_single_example(example, read_features)
--
GitLab
From 0c58c8f9c442ab2607a59e3d9e89f5aed4092ce1 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Tue, 13 Sep 2022 10:24:46 +0200
Subject: [PATCH 03/20] ADD: check existence of output_shapes.json and
output_types.json
---
otbtf/tfrecords.py | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 1acd01d0..7d1a4d4c 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -121,9 +121,6 @@ class TFRecords:
:param preprocessing_fn: Optional. A preprocessing function that process the input example
:param kwargs: some keywords arguments for preprocessing_fn
"""
- for file in [self.output_types_file, self.output_shapes_file]:
- assert self.output_types, f"The file {file} is missing!"
-
read_features = {key: tf.io.FixedLenFeature([], dtype=tf.string) for key in self.output_types}
example_parsed = tf.io.parse_single_example(example, read_features)
@@ -175,6 +172,9 @@ class TFRecords:
:param num_parallel_calls: number of parallel calls for the parsing + preprocessing step
:param kwargs: some keywords arguments for preprocessing_fn
"""
+ for file in [self.output_types_file, self.output_shapes_file]:
+ assert self.output_types, f"The file {file} is missing!"
+
options = tf.data.Options()
if shuffle_buffer_size:
options.experimental_deterministic = False # disable order, increase speed
--
GitLab
From b5eb85397937388624120f09218ea3ef127d896c Mon Sep 17 00:00:00 2001
From: Narcon Nicolas <nicolas.narcon@inrae.fr>
Date: Tue, 13 Sep 2022 13:12:00 +0200
Subject: [PATCH 04/20] Apply 1 suggestion(s) to 1 file(s)
---
otbtf/tfrecords.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 7d1a4d4c..254efd2a 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -172,8 +172,8 @@ class TFRecords:
:param num_parallel_calls: number of parallel calls for the parsing + preprocessing step
:param kwargs: some keywords arguments for preprocessing_fn
"""
- for file in [self.output_types_file, self.output_shapes_file]:
- assert self.output_types, f"The file {file} is missing!"
+ for dic, file in zip([self.output_types, self.output_shapes], [self.output_types_file, self.output_shapes_file]):
+ assert dic, f"The file {file} is missing!"
options = tf.data.Options()
if shuffle_buffer_size:
--
GitLab
From b44ae89fa5e52adefd035af7589430081d3b578c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 19 Sep 2022 18:57:30 +0200
Subject: [PATCH 05/20] DOC: bump version in setup.py
---
setup.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/setup.py b/setup.py
index 3a95ac4a..9725b146 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setuptools.setup(
name="otbtf",
- version="0.1",
+ version="3.3.0",
author="Remi Cresson",
author_email="remi.cresson@inrae.fr",
description="OTBTF: Orfeo ToolBox meets TensorFlow",
@@ -27,4 +27,4 @@ setuptools.setup(
packages=setuptools.find_packages(),
python_requires=">=3.6",
keywords="remote sensing, otb, orfeotoolbox, orfeo toolbox, tensorflow, tf, deep learning, machine learning",
-)
\ No newline at end of file
+)
--
GitLab
From d2195d84d251fd8c92c34e25c181be5e4cae37cd Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 19 Sep 2022 18:57:55 +0200
Subject: [PATCH 06/20] COMP: upgrade pip in dockerfile
---
Dockerfile | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 990c55f5..c48a01dc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -23,10 +23,12 @@ RUN if $GUI; then \
### Python3 links and pip packages
RUN ln -s /usr/bin/python3 /usr/local/bin/python && ln -s /usr/bin/pip3 /usr/local/bin/pip
+# Upgrade pip
+RUN pip install --no-cache-dir pip --upgrade
# NumPy version is conflicting with system's gdal dep and may require venv
ARG NUMPY_SPEC="==1.22.*"
ARG PROTO_SPEC="==3.20.*"
-RUN pip install --no-cache-dir -U pip wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" \
+RUN pip install --no-cache-dir -U wheel mock six future tqdm deprecated "numpy$NUMPY_SPEC" "protobuf$PROTO_SPEC" \
&& pip install --no-cache-dir --no-deps keras_applications keras_preprocessing
# ----------------------------------------------------------------------------
--
GitLab
From 705ec54538bf938884eb777f6215358afca4b728 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 19 Sep 2022 19:23:23 +0200
Subject: [PATCH 07/20] REFAC: move _is_chief() and cropped_tensor_name() into
model.py
---
otbtf/model.py | 35 ++++++++++++++++++++++++++++++++++-
otbtf/utils.py | 34 ----------------------------------
2 files changed, 34 insertions(+), 35 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index a6306e27..38fc6ca6 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -3,7 +3,6 @@
import abc
import logging
import tensorflow
-from otbtf.utils import _is_chief, cropped_tensor_name
class ModelBase(abc.ABC):
@@ -165,3 +164,37 @@ class ModelBase(abc.ABC):
model_simplified = tensorflow.keras.Model(inputs=inputs, outputs=outputs,
name=self.__class__.__name__ + '_simplified')
tensorflow.keras.utils.plot_model(model_simplified, output_path)
+
+
+def _is_chief(strategy):
+ """
+ Tell if the current worker is the chief.
+
+ :param strategy: strategy
+ :return: True if the current worker is the chief, False else
+ """
+ # Note: there are two possible `TF_CONFIG` configuration.
+ # 1) In addition to `worker` tasks, a `chief` task type is use;
+ # in this case, this function should be modified to
+ # `return task_type == 'chief'`.
+ # 2) Only `worker` task type is used; in this case, worker 0 is
+ # regarded as the chief. The implementation demonstrated here
+ # is for this case.
+ # For the purpose of this Colab section, the `task_type is None` case
+ # is added because it is effectively run with only a single worker.
+
+ if strategy.cluster_resolver: # this means MultiWorkerMirroredStrategy
+ task_type, task_id = strategy.cluster_resolver.task_type, strategy.cluster_resolver.task_id
+ return (task_type == 'chief') or (task_type == 'worker' and task_id == 0) or task_type is None
+ # strategy with only one worker
+ return True
+
+
+def cropped_tensor_name(tensor_name, crop):
+ """
+ A name for the padded tensor
+ :param tensor_name: tensor name
+ :param pad: pad value
+ :return: name
+ """
+ return "{}_crop{}".format(tensor_name, crop)
diff --git a/otbtf/utils.py b/otbtf/utils.py
index 7aa777af..069638a5 100644
--- a/otbtf/utils.py
+++ b/otbtf/utils.py
@@ -63,37 +63,3 @@ def read_as_np_arr(gdal_ds, as_patches=True, dtype=None):
buffer = buffer.astype(dtype)
return buffer
-
-
-def _is_chief(strategy):
- """
- Tell if the current worker is the chief.
-
- :param strategy: strategy
- :return: True if the current worker is the chief, False else
- """
- # Note: there are two possible `TF_CONFIG` configuration.
- # 1) In addition to `worker` tasks, a `chief` task type is use;
- # in this case, this function should be modified to
- # `return task_type == 'chief'`.
- # 2) Only `worker` task type is used; in this case, worker 0 is
- # regarded as the chief. The implementation demonstrated here
- # is for this case.
- # For the purpose of this Colab section, the `task_type is None` case
- # is added because it is effectively run with only a single worker.
-
- if strategy.cluster_resolver: # this means MultiWorkerMirroredStrategy
- task_type, task_id = strategy.cluster_resolver.task_type, strategy.cluster_resolver.task_id
- return (task_type == 'chief') or (task_type == 'worker' and task_id == 0) or task_type is None
- # strategy with only one worker
- return True
-
-
-def cropped_tensor_name(tensor_name, crop):
- """
- A name for the padded tensor
- :param tensor_name: tensor name
- :param pad: pad value
- :return: name
- """
- return "{}_crop{}".format(tensor_name, crop)
--
GitLab
From abd9d433ba746aac22e29b98b4118fd5053c512e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 19 Sep 2022 19:33:33 +0200
Subject: [PATCH 08/20] ADD: warning when otbtf is imported w/o GDAL
---
otbtf/__init__.py | 9 ++++++---
1 file changed, 6 insertions(+), 3 deletions(-)
diff --git a/otbtf/__init__.py b/otbtf/__init__.py
index 5321e3db..6ee17f9e 100644
--- a/otbtf/__init__.py
+++ b/otbtf/__init__.py
@@ -20,9 +20,12 @@
"""
OTBTF python module
"""
+try:
+ from otbtf.utils import read_as_np_arr, gdal_open
+ from otbtf.dataset import Buffer, PatchesReaderBase, PatchesImagesReader, IteratorBase, RandomIterator, Dataset, \
+ DatasetFromPatchesImages
+except ImportError:
+ print("Warning: otbtf.utils and otbtf.dataset were not imported. Using OTBTF without GDAL.")
-from otbtf.utils import read_as_np_arr, gdal_open
-from otbtf.dataset import Buffer, PatchesReaderBase, PatchesImagesReader, IteratorBase, RandomIterator, Dataset, \
- DatasetFromPatchesImages
from otbtf.tfrecords import TFRecords
from otbtf.model import ModelBase
--
GitLab
From 00b2f746cbd0a1d89d8a0441e0380b9cf4f86264 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 19 Sep 2022 19:39:46 +0200
Subject: [PATCH 09/20] ENH: better model plotting
---
otbtf/model.py | 17 +++++++++--------
1 file changed, 9 insertions(+), 8 deletions(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index a6306e27..f7e96779 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -145,23 +145,24 @@ class ModelBase(abc.ABC):
def summary(self, strategy=None):
"""
Wraps the summary printing of the model. When multiworker strategy, only prints if the worker is chief
+
+ :param strategy: strategy
"""
if not strategy or _is_chief(strategy):
self.model.summary(line_length=150)
- def plot(self, output_path, strategy=None):
+ def plot(self, output_path, strategy=None, show_shapes=False):
"""
Enables to save a figure representing the architecture of the network.
Needs pydot and graphviz to work (`pip install pydot` and https://graphviz.gitlab.io/download/)
+
+ :param output_path: output path for the schema
+ :param strategy: strategy
+ :param strategy: annotate with shapes values
+
"""
assert self.model, "Plot() only works if create_network() has been called beforehand"
# When multiworker strategy, only plot if the worker is chief
if not strategy or _is_chief(strategy):
- # Build a simplified model, without normalization nor extra outputs.
- # This model is only used for plotting the architecture thanks to `keras.utils.plot_model`
- inputs = self.get_inputs() # inputs without normalization
- outputs = self.get_outputs(inputs) # raw model outputs
- model_simplified = tensorflow.keras.Model(inputs=inputs, outputs=outputs,
- name=self.__class__.__name__ + '_simplified')
- tensorflow.keras.utils.plot_model(model_simplified, output_path)
+ tensorflow.keras.utils.plot_model(self.model, output_path, show_shapes=show_shapes)
--
GitLab
From d235e0c6037127c33280945fa5bd90daa052427d Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 19 Sep 2022 21:00:05 +0200
Subject: [PATCH 10/20] DOC: docstring
---
otbtf/model.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/otbtf/model.py b/otbtf/model.py
index f7e96779..91800dcd 100644
--- a/otbtf/model.py
+++ b/otbtf/model.py
@@ -158,7 +158,7 @@ class ModelBase(abc.ABC):
:param output_path: output path for the schema
:param strategy: strategy
- :param strategy: annotate with shapes values
+ :param show_shapes: annotate with shapes values (True or False)
"""
assert self.model, "Plot() only works if create_network() has been called beforehand"
--
GitLab
From 4c37af7b3337745de8c62505aa7c9d4e2824656d Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Mon, 19 Sep 2022 21:00:22 +0200
Subject: [PATCH 11/20] STYLE: pylint
---
otbtf/tfrecords.py | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/otbtf/tfrecords.py b/otbtf/tfrecords.py
index 254efd2a..15a23354 100644
--- a/otbtf/tfrecords.py
+++ b/otbtf/tfrecords.py
@@ -172,7 +172,8 @@ class TFRecords:
:param num_parallel_calls: number of parallel calls for the parsing + preprocessing step
:param kwargs: some keywords arguments for preprocessing_fn
"""
- for dic, file in zip([self.output_types, self.output_shapes], [self.output_types_file, self.output_shapes_file]):
+ for dic, file in zip([self.output_types, self.output_shapes],
+ [self.output_types_file, self.output_shapes_file]):
assert dic, f"The file {file} is missing!"
options = tf.data.Options()
--
GitLab
From 3b6a77e8f421f9633dc5ace611c8b07675e737ce Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 21 Sep 2022 15:50:15 +0200
Subject: [PATCH 12/20] COMP: use OTB 8.1.0
---
Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index c48a01dc..7dabf6fb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -87,7 +87,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
### OTB
ARG GUI=false
-ARG OTB=8.0.1
+ARG OTB=8.1.0
ARG OTBTESTS=false
RUN mkdir /src/otb
--
GitLab
From 9b4496f3f43c67904a31489c62c1d68fe1db847b Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 21 Sep 2022 15:51:27 +0200
Subject: [PATCH 13/20] Revert "COMP: use OTB 8.1.0"
This reverts commit 3b6a77e8f421f9633dc5ace611c8b07675e737ce.
---
Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index 7dabf6fb..c48a01dc 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -87,7 +87,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
### OTB
ARG GUI=false
-ARG OTB=8.1.0
+ARG OTB=8.0.1
ARG OTBTESTS=false
RUN mkdir /src/otb
--
GitLab
From 24feed60ec86ec3fd112c8d44497d2f0ba48e713 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Wed, 21 Sep 2022 15:52:16 +0200
Subject: [PATCH 14/20] COMP: use OTB 8.1.0
---
Dockerfile | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/Dockerfile b/Dockerfile
index c48a01dc..7dabf6fb 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -87,7 +87,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
### OTB
ARG GUI=false
-ARG OTB=8.0.1
+ARG OTB=8.1.0
ARG OTBTESTS=false
RUN mkdir /src/otb
--
GitLab
From a66ec85dbcc17f1913b8b6e2353a2c2ccdc8f09c Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 22 Sep 2022 10:43:28 +0200
Subject: [PATCH 15/20] DOC: update release notes
---
RELEASE_NOTES.txt | 9 +++++++++
1 file changed, 9 insertions(+)
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 5d78c0d7..47bfb990 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -1,3 +1,12 @@
+Version 3.3.2 (22 sep 2022)
+----------------------------------------------------------------
+* `otbtf.TFRecords` can be used in environments where only TensorFlow is present, without GDAL or OTB (e.g. HPC centers).
+* Simplify the `plot()` method in the `otbtf.ModelBase` class
+* Upgrade OTB to version 8.1.0
+* Build GDAL with GEOS support
+* Add a warning message in `TFRecords` when output_shape or output_type is missing
+* Upgrade pip in the docker images
+
Version 3.3.0 (27 jul 2022)
----------------------------------------------------------------
* Improves the `dataset` classes (`DatasetFromPatchesImages`, `TFRecords`) to use them easily in keras
--
GitLab
From d963dc547cc40a5962240c5b8e3b308d23abe3e6 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 22 Sep 2022 10:44:23 +0200
Subject: [PATCH 16/20] CI: bump OTBTF version to 3.3.2
---
.gitlab-ci.yml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 03a377d4..2e25927a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,7 +1,7 @@
image: $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
variables:
- OTBTF_VERSION: 3.3.0
+ OTBTF_VERSION: 3.3.2
OTB_BUILD: /src/otb/build/OTB/build # Local OTB build directory
OTBTF_SRC: /src/otbtf # Local OTBTF source directory
OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary # OTB testing directory
--
GitLab
From 0fb8786687ea872b175ea84fdf0d0cd3c3e1c41e Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 22 Sep 2022 11:22:31 +0200
Subject: [PATCH 17/20] DOC: update docker images list
---
doc/DOCKERUSE.md | 37 ++++++++++++++++++++++---------------
1 file changed, 22 insertions(+), 15 deletions(-)
diff --git a/doc/DOCKERUSE.md b/doc/DOCKERUSE.md
index 50e505de..201b5b71 100644
--- a/doc/DOCKERUSE.md
+++ b/doc/DOCKERUSE.md
@@ -7,12 +7,12 @@ Since OTBTF >= 3.2.1 you can find latest docker images on [gitlab.irstea.fr](htt
| Name | Os | TF | OTB | Description | Dev files | Compute capability |
|------------------------------------------------------------------------------------| ------------- | ------ |-------| ---------------------- | --------- | ------------------ |
-| **mdl4eo/otbtf:3.3.0-cpu** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.3.0-cpu-dev** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.3.0-gpu** | Ubuntu Focal | r2.8 | 8.0.1 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.3.0-gpu-dev** | Ubuntu Focal | r2.8 | 8.0.1 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt** | Ubuntu Focal | r2.8 | 8.0.1 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
-| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.0.1 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.2-cpu** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.2-cpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.2-gpu** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.2-gpu-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.2-gpu-opt** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.2-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.1.0 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
The list of older releases is available [here](#older-docker-releases).
@@ -37,7 +37,7 @@ For instance, suppose you have some data in `/mnt/my_device/` that you want to u
The following command shows you how to access the folder from the docker image.
```bash
-docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf:3.3.0-cpu bash -c "ls /data"
+docker run -v /mnt/my_device/:/data/ -ti mdl4eo/otbtf:3.3.2-cpu bash -c "ls /data"
```
Beware of ownership issues! see the last section of this doc.
@@ -50,13 +50,13 @@ You can then use the OTBTF `gpu` tagged docker images with the **NVIDIA runtime*
With Docker version earlier than 19.03 :
```bash
-docker run --runtime=nvidia -ti mdl4eo/otbtf:3.3.0-gpu bash
+docker run --runtime=nvidia -ti mdl4eo/otbtf:3.3.2-gpu bash
```
With Docker version including and after 19.03 :
```bash
-docker run --gpus all -ti mdl4eo/otbtf:3.3.0-gpu bash
+docker run --gpus all -ti mdl4eo/otbtf:3.3.2-gpu bash
```
You can find some details on the **GPU docker image** and some **docker tips and tricks** on [this blog](https://mdl4eo.irstea.fr/2019/10/15/otbtf-docker-image-with-gpu/).
@@ -69,7 +69,7 @@ Be careful though, these infos might be a bit outdated...
1. Install [WSL2](https://docs.microsoft.com/en-us/windows/wsl/install-win10#manual-installation-steps) (Windows Subsystem for Linux)
2. Install [docker desktop](https://www.docker.com/products/docker-desktop)
3. Start **docker desktop** and **enable WSL2** from *Settings* > *General* then tick the box *Use the WSL2 based engine*
-3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf:3.3.0-cpu`
+3. Open a **cmd.exe** or **PowerShell** terminal, and type `docker create --name otbtf-cpu --interactive --tty mdl4eo/otbtf:3.3.2-cpu`
4. Open **docker desktop**, and check that the docker is running in the **Container/Apps** menu
5. From **docker desktop**, click on the icon highlighted as shown below, and use the bash terminal that should pop up!
@@ -118,12 +118,12 @@ sudo systemctl {status,enable,disable,start,stop} docker
Run a simple command in a one-shot container:
```bash
-docker run mdl4eo/otbtf:3.3.0-cpu otbcli_PatchesExtraction
+docker run mdl4eo/otbtf:3.3.2-cpu otbcli_PatchesExtraction
```
You can also use the image in interactive mode with bash:
```bash
-docker run -ti mdl4eo/otbtf:3.3.0-cpu bash
+docker run -ti mdl4eo/otbtf:3.3.2-cpu bash
```
### Persistent container
@@ -133,7 +133,7 @@ Beware of ownership issues, see the last section of this doc.
```bash
docker create --interactive --tty --volume /home/$USER:/home/otbuser/ \
- --name otbtf mdl4eo/otbtf:3.3.0-cpu /bin/bash
+ --name otbtf mdl4eo/otbtf:3.3.2-cpu /bin/bash
```
### Interactive session
@@ -197,7 +197,7 @@ Create a named container (here with your HOME as volume), Docker will automatica
```bash
docker create --interactive --tty --volume /home/$USER:/home/otbuser \
- --name otbtf mdl4eo/otbtf:3.3.0-cpu /bin/bash
+ --name otbtf mdl4eo/otbtf:3.3.2-cpu /bin/bash
```
Start a background container process:
@@ -264,4 +264,11 @@ Here you can find the list of older releases of OTBTF:
| **mdl4eo/otbtf:3.1-gpu** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.1-gpu-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
| **mdl4eo/otbtf:3.1-gpu-opt** | Ubuntu Focal | r2.8 | 7.4.0 | GPU | no | 5.2,6.1,7.0,7.5,8.6|
-| **mdl4eo/otbtf:3.1-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
\ No newline at end of file
+| **mdl4eo/otbtf:3.1-gpu-opt-dev** | Ubuntu Focal | r2.8 | 7.4.0 | GPU (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-cpu** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-cpu-dev** | Ubuntu Focal | r2.8 | 8.0.1 | CPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-gpu** | Ubuntu Focal | r2.8 | 8.0.1 | GPU, no optimization | no | 5.2,6.1,7.0,7.5,8.6|
+| **mdl4eo/otbtf:3.3.0-gpu-dev** | Ubuntu Focal | r2.8 | 8.0.1 | GPU, no optimization (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt** | Ubuntu Focal | r2.8 | 8.0.1 | GPU with opt. | no | 5.2,6.1,7.0,7.5,8.6|
+| **gitlab.irstea.fr/remi.cresson/otbtf/container_registry/otbtf:3.3.0-gpu-opt-dev** | Ubuntu Focal | r2.8 | 8.0.1 | GPU with opt. (dev) | yes | 5.2,6.1,7.0,7.5,8.6|
+
--
GitLab
From 316135c6f5e97dfd8beefe3e2841fc549b0ba1e5 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 22 Sep 2022 11:22:58 +0200
Subject: [PATCH 18/20] DOC: update version
---
README.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/README.md b/README.md
index c2a13a9c..fe4e305a 100644
--- a/README.md
+++ b/README.md
@@ -55,7 +55,7 @@ For now you have two options: either use the existing **docker image**, or build
Use the latest CPU or GPU-enabled image from dockerhub:
```
-docker run mdl4eo/otbtf:3.3.0-cpu otbcli_PatchesExtraction -help
+docker run mdl4eo/otbtf:3.3.2-cpu otbcli_PatchesExtraction -help
```
Read more in the [docker use documentation](doc/DOCKERUSE.md).
--
GitLab
From b78ec22cbca854387448c3e74140c5b9ee17aeed Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 22 Sep 2022 11:23:13 +0200
Subject: [PATCH 19/20] COMP: 3.3.0 --> 3.3.2
---
setup.py | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/setup.py b/setup.py
index 9725b146..9a7f6eb0 100644
--- a/setup.py
+++ b/setup.py
@@ -6,7 +6,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
setuptools.setup(
name="otbtf",
- version="3.3.0",
+ version="3.3.2",
author="Remi Cresson",
author_email="remi.cresson@inrae.fr",
description="OTBTF: Orfeo ToolBox meets TensorFlow",
--
GitLab
From 80a84efaf3851947fc47774968bfc813b93d5955 Mon Sep 17 00:00:00 2001
From: Remi Cresson <remi.cresson@inrae.fr>
Date: Thu, 22 Sep 2022 11:23:25 +0200
Subject: [PATCH 20/20] DOC: update docker images list
---
tools/docker/README.md | 36 ++++++++++++++++++++----------------
1 file changed, 20 insertions(+), 16 deletions(-)
diff --git a/tools/docker/README.md b/tools/docker/README.md
index dc718e68..3dcf38f8 100644
--- a/tools/docker/README.md
+++ b/tools/docker/README.md
@@ -1,8 +1,8 @@
# Build with Docker
-Docker build has to be called from the root of the repository (i.e. `docker build .` or `bash tools/docker/multibuild.sh`).
+Docker build has to be called from the root of the repository (i.e. `docker build .` or `bash tools/docker/multibuild.sh`).
You can build a custom image using `--build-arg` and several config files :
- Ubuntu : `BASE_IMG` should accept any version, for additional packages see [build-deps-cli.txt](build-deps-cli.txt) and [build-deps-gui.txt](build-deps-gui.txt).
-- TensorFlow : `TF` arg for the git branch or tag + [build-env-tf.sh](build-env-tf.sh) and BZL_* arguments for the build configuration. `ZIP_TF_BIN` allows you to save compiled binaries if you want to install it elsewhere.
+- TensorFlow : `TF` arg for the git branch or tag + [build-env-tf.sh](build-env-tf.sh) and BZL_* arguments for the build configuration. `ZIP_TF_BIN` allows you to save compiled binaries if you want to install it elsewhere.
- OrfeoToolBox : `OTB` arg for the git branch or tag + [build-flags-otb.txt](build-flags-otb.txt) to edit cmake flags. Set `KEEP_SRC_OTB` in order to preserve OTB git directory.
### Base images
@@ -20,8 +20,8 @@ BASE_IMG # mandatory
CPU_RATIO=1
GUI=false
NUMPY_SPEC="==1.19.*"
-TF=v2.5.0
-OTB=7.3.0
+TF=v2.8.0
+OTB=8.1.0
BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package"
BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
@@ -32,13 +32,14 @@ SUDO=true
# NumPy version requirement :
# TF < 2.4 : "numpy<1.19.0,>=1.16.0"
# TF >= 2.4 : "numpy==1.19.*"
+# TF >= 2.8 : "numpy==1.22.*"
```
### Bazel remote cache daemon
-If you just need to rebuild with different GUI or KEEP_SRC arguments, or may be a different branch of OTB, bazel cache will help you to rebuild everything except TF, even if the docker cache was purged (after `docker [system|builder] prune`).
-In order to recycle the cache, bazel config and TF git tag should be exactly the same, any change in [build-env-tf.sh](build-env-tf.sh) and `--build-arg` (if related to bazel env, cuda, mkl, xla...) may result in a fresh new build.
+If you just need to rebuild with different GUI or KEEP_SRC arguments, or may be a different branch of OTB, bazel cache will help you to rebuild everything except TF, even if the docker cache was purged (after `docker [system|builder] prune`).
+In order to recycle the cache, bazel config and TF git tag should be exactly the same, any change in [build-env-tf.sh](build-env-tf.sh) and `--build-arg` (if related to bazel env, cuda, mkl, xla...) may result in a fresh new build.
-Start a cache daemon - here with max 20GB but 10GB should be enough to save 2 TF builds (GPU and CPU):
+Start a cache daemon - here with max 20GB but 10GB should be enough to save 2 TF builds (GPU and CPU):
```bash
mkdir -p $HOME/.cache/bazel-remote
docker run --detach -u 1000:1000 -v $HOME/.cache/bazel-remote:/data -p 9090:8080 buchgr/bazel-remote-cache --max_size=20
@@ -73,6 +74,9 @@ docker build --network='host' -t otbtf:oldstable-gpu --build-arg BASE_IMG=nvidia
```
### Build for another machine and save TF compiled files
+
+Example with TF 2.5
+
```bash
# Use same ubuntu and CUDA version than your target machine, beware of CC optimization and CPU compatibility
# (set env variable CC_OPT_FLAGS and avoid "-march=native" if your Docker's CPU is optimized with AVX2/AVX512 but your target CPU isn't)
@@ -104,14 +108,14 @@ cmake $OTB_GIT \
```
### Debug build
-If you fail to build, you can log into the last layer and check CMake logs. Run `docker images`, find the latest layer ID and run a tmp container (`docker run -it d60496d9612e bash`).
-You may also need to split some multi-command layers in the Dockerfile.
-If you see OOM errors during SuperBuild you should decrease CPU_RATIO (e.g. 0.75).
+If you fail to build, you can log into the last layer and check CMake logs. Run `docker images`, find the latest layer ID and run a tmp container (`docker run -it d60496d9612e bash`).
+You may also need to split some multi-command layers in the Dockerfile.
+If you see OOM errors during SuperBuild you should decrease CPU_RATIO (e.g. 0.75).
## Container examples
```bash
# Pull GPU image and create a new container with your home directory as volume (requires apt package nvidia-docker2 and CUDA>=11.0)
-docker create --gpus=all --volume $HOME:/home/otbuser/volume -it --name otbtf-gpu mdl4eo/otbtf:3.3.0-gpu
+docker create --gpus=all --volume $HOME:/home/otbuser/volume -it --name otbtf-gpu mdl4eo/otbtf:3.3.2-gpu
# Run interactive
docker start -i otbtf-gpu
@@ -123,7 +127,7 @@ docker exec otbtf-gpu python -c 'import tensorflow as tf; print(tf.test.is_gpu_a
### Rebuild OTB with more modules
```bash
-docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf:3.3.0-gpu-dev
+docker create --gpus=all -it --name otbtf-gpu-dev mdl4eo/otbtf:3.3.2-gpu-dev
docker start -i otbtf-gpu-dev
```
```bash
@@ -146,10 +150,10 @@ $ mapla
```
## Common errors
-Build :
-`Error response from daemon: manifest for nvidia/cuda:11.0-cudnn8-devel-ubuntu20.04 not found: manifest unknown: manifest unknown`
+Build :
+`Error response from daemon: manifest for nvidia/cuda:11.0-cudnn8-devel-ubuntu20.04 not found: manifest unknown: manifest unknown`
=> Image is missing from dockerhub
-Run :
-`failed call to cuInit: UNKNOWN ERROR (303) / no NVIDIA GPU device is present: /dev/nvidia0 does not exist`
+Run :
+`failed call to cuInit: UNKNOWN ERROR (303) / no NVIDIA GPU device is present: /dev/nvidia0 does not exist`
=> Nvidia driver is missing or disabled, make sure to add ` --gpus=all` to your docker run or create command
--
GitLab