Add scriptmodule creation option to cmake (#62)

Author: mfl28

.gitignore

@@ -61,6 +61,7 @@ cmake-build-*/
 # Other
 .DS_Store
 .idea/
+.clangd/
 
 # Notebooks
 .ipynb_checkpoints

CMakeLists.txt

@@ -5,6 +5,7 @@ project(pytorch-cpp VERSION 1.0.0 LANGUAGES CXX)
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
 
 option(DOWNLOAD_DATASETS "Download datasets used in the tutorials." ON)
+option(CREATE_SCRIPTMODULES "Create all necessary scriptmodule files." OFF)
 
 set(PYTORCH_VERSION "1.6.0")
 
@@ -19,6 +20,10 @@ if(DOWNLOAD_DATASETS)
     include(download_datasets)
 endif()
 
+if(CREATE_SCRIPTMODULES)
+    find_package(Python3 COMPONENTS Interpreter REQUIRED)
+endif()
+
 set(EXECUTABLE_NAME pytorch-cpp)
 
 add_executable(${EXECUTABLE_NAME})

README.md

@@ -11,7 +11,7 @@
 </p>
 
 
-| OS (Compiler)\\libtorch |                                                  1.6.0                                                  |  nightly |
+| OS (Compiler)\\LibTorch |                                                  1.6.0                                                  |  nightly |
 | :---------------------: | :---------------------------------------------------------------------------------------------------: |  :-----: |
 |    macOS (clang 9.1)    | ![Status](https://travis-matrix-badges.herokuapp.com/repos/prabhuomkar/pytorch-cpp/branches/master/1) |          |
 |    macOS (clang 10.0)   | ![Status](https://travis-matrix-badges.herokuapp.com/repos/prabhuomkar/pytorch-cpp/branches/master/2) |          |
@@ -58,7 +58,7 @@ cd pytorch-cpp
 
 #### Generate build system
 
-```cmake
+```bash
 cmake -B build #<options>
 ```
 > **_Note for Windows users:_**<br> 
@@ -68,18 +68,50 @@ Some useful options:
 
 | Option       | Default           | Description  |
 | :------------- |:------------|-----:|
-| `-D CUDA_V=(9.2 [Linux only]\|10.1\|10.2\|none)`     | `none` | Download libtorch for a CUDA version (`none` = download CPU version). |
+| `-D CUDA_V=(9.2 [Linux only]\|10.1\|10.2\|none)`     | `none` | Download LibTorch for a CUDA version (`none` = download CPU version). |
 | `-D DOWNLOAD_DATASETS=(OFF\|ON)`     | `ON`      |   Download all datasets used in the tutorials. |
-| `-D CMAKE_PREFIX_PATH=path/to/libtorch/share/cmake/Torch` |   `<empty>`    |    Skip the downloading of libtorch and use your own local version (see Requirements) instead. |
-| `-D CMAKE_BUILD_TYPE=(Release\|Debug)` | `<empty>` (`Release` when downloading libtorch on Windows) | Set the build type (`Release` = compile with optimizations)|
+| `-D CMAKE_PREFIX_PATH=path/to/libtorch/share/cmake/Torch` |   `<empty>`    |    Skip the downloading of LibTorch and use your own local version (see [Requirements](#requirements)) instead. |
+| `-D CMAKE_BUILD_TYPE=(Release\|Debug)` | `<empty>` (`Release` when downloading LibTorch on Windows) | Set the build type (`Release` = compile with optimizations).|
+|`-D CREATE_SCRIPTMODULES=(OFF\|ON)` | `OFF` | Create all needed scriptmodule files for prelearned models / weights. Requires installed  python3 with  pytorch and torchvision. |
+
+<details>
+<summary><b>Example Linux</b></summary>
+
+##### Aim
+* Use existing Python, PyTorch (see [Requirements](#requirements)) and torchvision installation.
+* Download all datasets and create all necessary scriptmodule files.
+
+##### Command
+```bash
+cmake -B build \
+-D CMAKE_BUILD_TYPE=Release \
+-D CMAKE_PREFIX_PATH=/path/to/libtorch/share/cmake/Torch \
+-D CREATE_SCRIPTMODULES=ON 
+```
+</details>
+
+<details>
+<summary><b>Example Windows</b></summary>
+
+##### Aim
+* Automatically download LibTorch for CUDA 10.2 and all necessary datasets.
+* Do not create scriptmodule files.
+
+##### Command
+```bash
+cmake -B build \
+-A x64 \
+-D CUDA_V=10.2
+```
+</details>
 
 #### Build
 
-```cmake
+```bash
 cmake --build build
 ```
 >**_Note for Windows users:_** <br>
->The CMake script downloads the *Release* version of libtorch, so `--config Release` has to be appended to the build command.
+>The CMake script downloads the *Release* version of LibTorch, so `--config Release` has to be appended to the build command.
 >
 >**_General Note:_** <br>
 >By default all tutorials will be built. If you only want to build  one specific tutorial, specify the `target` parameter for the build command. For example to only build the language model tutorial, append `--target language-model` (target name = tutorial foldername with all underscores replaced with hyphens).
@@ -126,4 +158,4 @@ cmake --build build
 * [Tensor Slicing](notebooks/tensor_slicing.ipynb)
 
 ## License
-This repository is licensed under MIT as given in [LICENSE](LICENSE).
+This repository is licensed under MIT as given in [LICENSE](LICENSE).

cmake/download_flickr8k.cmake

@@ -7,25 +7,25 @@ set(FLICKR8K_DIR "${CMAKE_SOURCE_DIR}/data/flickr_8k")
 set(FLICKR8K_DATA_URL "https://github.com/jbrownlee/Datasets/releases/download/Flickr8k/Flickr8k_Dataset.zip")
 set(FLICKR8K_TEXT_URL "https://github.com/jbrownlee/Datasets/releases/download/Flickr8k/Flickr8k_text.zip")
 
-set(FLICKR8k_DATA_SOURCE_DIR "${FLICKR8K_DIR}/Flickr8k_Dataset")
-set(FLICKR8k_TEXT_SOURCE_DIR "${FLICKR8K_DIR}/Flickr8k_text")
+set(FLICKR8K_DATA_SOURCE_DIR "${FLICKR8K_DIR}/Flickr8k_Dataset")
+set(FLICKR8K_TEXT_SOURCE_DIR "${FLICKR8K_DIR}/Flickr8k_text")
 
-if(NOT EXISTS ${FLICKR8k_TEXT_SOURCE_DIR} OR NOT EXISTS ${FLICKR8k_DATA_SOURCE_DIR})
+if(NOT EXISTS ${FLICKR8K_TEXT_SOURCE_DIR} OR NOT EXISTS ${FLICKR8K_DATA_SOURCE_DIR})
     message(STATUS "Fetching Flickr8k dataset...")
 
     FetchContent_Declare(
         flickr_8k_text
         DOWNLOAD_DIR ${FLICKR8K_DIR}/download
-        SOURCE_DIR ${FLICKR8k_TEXT_SOURCE_DIR}
+        SOURCE_DIR ${FLICKR8K_TEXT_SOURCE_DIR}
         URL ${FLICKR8K_TEXT_URL}
     )
 
     FetchContent_MakeAvailable(flickr_8k_text)
-
+    set(FETCHCONTENT_QUIET OFF CACHE BOOL "" FORCE)
     FetchContent_Declare(
         flickr_8k_data
         DOWNLOAD_DIR ${FLICKR8K_DIR}/download
-        SOURCE_DIR ${FLICKR8k_DATA_SOURCE_DIR}
+        SOURCE_DIR ${FLICKR8K_DATA_SOURCE_DIR}
         URL ${FLICKR8K_DATA_URL}
     )
 
@@ -36,4 +36,4 @@ if(NOT EXISTS ${FLICKR8k_TEXT_SOURCE_DIR} OR NOT EXISTS ${FLICKR8k_DATA_SOURCE_D
     message(STATUS "Fetching Flickr8k dataset - done")
 else()
     message(STATUS "Flickr8k dataset already present, skipping...")
-endif()
+endif()

tutorials/advanced/image_captioning/CMakeLists.txt

@@ -41,10 +41,15 @@ set_target_properties(${EXECUTABLE_NAME} PROPERTIES
 add_custom_command(TARGET ${EXECUTABLE_NAME}
                    POST_BUILD
                    COMMAND ${CMAKE_COMMAND} -E make_directory output)
+               
+if(CREATE_SCRIPTMODULES AND NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/model/encoder_cnn_backbone.pt")
+    execute_process(COMMAND ${Python3_EXECUTABLE} "create_encoder_cnn_backbone_scriptmodule.py" 
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/model")
+endif()
 
 if(MSVC)
     set_target_properties(${EXECUTABLE_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR})
     set_target_properties(${EXECUTABLE_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR})
     include(copy_torch_dlls)
     copy_torch_dlls(${EXECUTABLE_NAME})
-endif(MSVC)
+endif(MSVC)

tutorials/advanced/neural_style_transfer/CMakeLists.txt

@@ -27,9 +27,14 @@ add_custom_command(TARGET ${EXECUTABLE_NAME}
                    POST_BUILD
                    COMMAND ${CMAKE_COMMAND} -E make_directory output)
 
+if(CREATE_SCRIPTMODULES AND NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/model/vgg19_layers.pt")
+    execute_process(COMMAND ${Python3_EXECUTABLE} "create_vgg19_layers_scriptmodule.py" 
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/model")
+endif()
+
 if(MSVC)
     set_target_properties(${EXECUTABLE_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR})
     set_target_properties(${EXECUTABLE_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR})
     include(copy_torch_dlls)
     copy_torch_dlls(${EXECUTABLE_NAME})
-endif(MSVC)
+endif(MSVC)

tutorials/basics/pytorch_basics/CMakeLists.txt

@@ -23,9 +23,14 @@ add_custom_command(TARGET ${EXECUTABLE_NAME}
                    POST_BUILD
                    COMMAND ${CMAKE_COMMAND} -E make_directory output)
 
+if(CREATE_SCRIPTMODULES AND NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/model/resnet18_scriptmodule.pt")
+    execute_process(COMMAND ${Python3_EXECUTABLE} "create_resnet18_scriptmodule.py" 
+        WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}/model")
+endif()
+
 if(MSVC)
     set_target_properties(${EXECUTABLE_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_RELEASE ${CMAKE_CURRENT_BINARY_DIR})
     set_target_properties(${EXECUTABLE_NAME} PROPERTIES RUNTIME_OUTPUT_DIRECTORY_DEBUG ${CMAKE_CURRENT_BINARY_DIR})
     include(copy_torch_dlls)
     copy_torch_dlls(${EXECUTABLE_NAME})
-endif(MSVC)
+endif(MSVC)

tutorials/basics/pytorch_basics/main.cpp

@@ -232,8 +232,8 @@ int main() {
     // Path to serialized ScriptModule of pretrained resnet18 model,
     // created in Python.
     // You can use the provided Python-script "create_resnet18_scriptmodule.py" in
-    // tutorials/basics/pytorch-basics/models to create the necessary file.
-    const std::string pretrained_model_path = "../../../../tutorials/basics/pytorch_basics/models/"
+    // tutorials/basics/pytorch-basics/model to create the necessary file.
+    const std::string pretrained_model_path = "../../../../tutorials/basics/pytorch_basics/model/"
         "resnet18_scriptmodule.pt";
 
     torch::jit::script::Module resnet;
@@ -244,7 +244,7 @@ int main() {
     catch (const torch::Error& error) {
         std::cerr << "Could not load scriptmodule from file " << pretrained_model_path << ".\n"
             << "You can create this file using the provided Python script 'create_resnet18_scriptmodule.py' "
-            "in tutorials/basics/pytorch-basics/models/.\n";
+            "in tutorials/basics/pytorch-basics/model/.\n";
         return -1;
     }