ubuntu16.04 + cuda9.0+cudnn7.3.1

2020-10-22 本文已影响0人 CrazyWolf_081c

ubuntu16.04 + cuda9.0+cudnn7.3.1 安装densepose

[toc]

准备工作

提前按照好CUDA和cudnn
使用Anaconda创建densepose环境

conda  create  -n densepose  python=2.7

开启虚拟环境

source  activate  densepose   这就开启了

第一步安装pytorch

这个pytorch中已经包括了caffe2的，直接在虚拟环境中安装pytorch：

conda install pytorch-nightly -c pytorch

测试安装的caffe2：

python2 -c 'from caffe2.python import core' 2>/dev/null && echo "Success" || echo "Failure"
Success

python2 -c 'from caffe2.python import workspace; print(workspace.NumCudaDevices())'
1

以上就表明caffe2安装成功，只要没有问题就往下做。

测试时错误1：ImportError: No module named google.protobuf.internal

测试时错误2：ImportError: No module named past.builtins

可以通过安装下面常用的包解决：

conda install cython
conda install protobuf==3.6.1
conda install future

第二步安装detectorn

下载源代码：

git clone https://github.com/facebookresearch/detectron

在densepose虚拟环境下安装：

pip install -r $DETECTRON/requirements.txt

安装opencv-python错误，直接pip安装会安装最新的4.3，导致出错，手动安装opencv-python：

pip install opencv-python==3.2.0.6

编译源代码：

cd detectron && make 直接编译成功

测试安装的detectron

python  ./detectron/tests/test_spatial_narrow_as_op.py   出现ok表明安装成功哦，

第三步安装cocoapi

下载源文件：

git clone https://github.com/cocodataset/cocoapi.git

到对应的PythonAPI的路径：

cd   cocoapi/PythonAPI

编译：

make  install

安装到环境中：

python2 setup.py install --user

第四步安装densepose（hard级别）

下载源代码：

git clone https://github.com/facebookresearch/densepose

安装依赖：

pip install -r $DENSEPOSE/requirements.txt

编译源代码：

sudo make

错误：fataerror : python.h找不到
解决方案：

sudo apt-get  install python-dev 

再次运行 sudo  make

验证make安装：

python ./detectron/tests/test_spatial_narrow_as_op.py

错误：找不到caffe2, $DENSEPOSE/detectron/utils/env.py 在这个文件中 AssertionError: Detectron ops lib not found; make sure that your Caffe2 version includes Detectron module

解决方案参考： https://zhuanlan.zhihu.com/p/104395486

找不到：libcaffe2_detectron_ops_gpu.so，先找到这个东西在哪里

1. 执行命令：sudo find / -name libcaffe2_detectron_ops_gpu.so
2. 找到自己torch的那个路径把：/MY/PATH/.conda/envs/dense2/lib/python2.7/site-packages/torch/ 加入到python的环境变中中，sys.path.append('/MY/PATH/.conda/envs/dense2/lib/python2.7/site-packages/torch/')
3. 然后在env.py文件中：修改 prefixes = [_CMAKE_INSTALL_PREFIX, sys.prefix, sys.exec_prefix] + sys.path + ['/MY/PATH/.conda/envs/dense2/lib/python2.7/site-packages/torch/']
4. 保存
5. 运行：python ./detectron/tests/test_spatial_narrow_as_op.py 成功

再次测试，测试成功！！！

make ops(最大bug出来了)

为防止逐个踩坑，现直接掏出生化武器，直接修改cmakelist.txt，可以解决好几个小bug：

cmake_minimum_required(VERSION 2.8.12 FATAL_ERROR)

set(Caffe2_DIR "/opt/conda/envs/densepose/lib/python2.7/site-packages/torch/share/cmake/Caffe2")

include_directories("/opt/conda/envs/densepose/include")
include_directories("/opt/caffe/external/mkl/mklml_lnx_2019.0.20180710/include/")
add_library(libprotobuf STATIC IMPORTED)
set(PROTOBUF_LIB "/opt/conda/envs/densepose/lib/libprotobuf.a")
set_property(TARGET libprotobuf PROPERTY IMPORTED_LOCATION "${PROTOBUF_LIB}")

# Find the Caffe2 package.
# Caffe2 exports the required targets, so find_package should work for
# the standard Caffe2 installation. If you encounter problems with finding
# the Caffe2 package, make sure you have run `make install` when installing
# Caffe2 (`make install` populates your share/cmake/Caffe2).
find_package(Caffe2 REQUIRED)
include_directories(${CAFFE2_INCLUDE_DIRS})

if (${CAFFE2_VERSION} VERSION_LESS 0.8.2)
  # Pre-0.8.2 caffe2 does not have proper interface libraries set up, so we
  # will rely on the old path.
  message(WARNING
      "You are using an older version of Caffe2 (version " ${CAFFE2_VERSION}
      "). Please consider moving to a newer version.")
  include(cmake/legacy/legacymake.cmake)
  return()
endif()

# Add compiler flags.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -std=c11")
set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -O2 -fPIC -Wno-narrowing")

# Print configuration summary.
include(cmake/Summary.cmake)
detectron_print_config_summary()

# Collect custom ops sources.
file(GLOB CUSTOM_OPS_CPU_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/detectron/ops/*.cc)
file(GLOB CUSTOM_OPS_GPU_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/detectron/ops/*.cu)

# Install custom CPU ops lib.
add_library(
     caffe2_detectron_custom_ops SHARED
     ${CUSTOM_OPS_CPU_SRCS})

target_link_libraries(caffe2_detectron_custom_ops caffe2_library libprotobuf)
install(TARGETS caffe2_detectron_custom_ops DESTINATION lib)

# Install custom GPU ops lib, if gpu is present.
if (CAFFE2_USE_CUDA OR CAFFE2_FOUND_CUDA)
  # Additional -I prefix is required for CMake versions before commit (< 3.7):
  # https://github.com/Kitware/CMake/commit/7ded655f7ba82ea72a82d0555449f2df5ef38594
  list(APPEND CUDA_INCLUDE_DIRS -I${CAFFE2_INCLUDE_DIRS})
  CUDA_ADD_LIBRARY(
      caffe2_detectron_custom_ops_gpu SHARED
      ${CUSTOM_OPS_CPU_SRCS}
      ${CUSTOM_OPS_GPU_SRCS})

  target_link_libraries(caffe2_detectron_custom_ops_gpu caffe2_gpu_library libprotobuf)
  install(TARGETS caffe2_detectron_custom_ops_gpu DESTINATION lib)
endif()

还剩下几个比较坚持的bug，下面一一进行解决。

错误1: make ops到了50%出错了，protobuf出问题

参考https://blog.csdn.net/FatMigo/article/details/88648107 分成四步完成

cp -r /home/xwt/.conda/pkgs/libprotobuf-3.6.1-hd408876_0/include/google      /home/xwt/anaconda3/include
cp -r  /home/xwt/.conda/pkgs/libprotobuf-3.6.1-hd408876_0/lib/libprotobuf*   /home/xwt/anaconda3/lib

protobuf的问题：解决了

错误2：fatal error: mkl_cblas.h: No such file or directory

bug解决链接： http://linkinpark213.com/2018/11/18/densepose-minesweeping/#2-3-cmake-files-not-found-amp-Unknown-CMake-command-quot-caffe2-interface-library-quot

查找：

sudo find / -name mkl_cblas.h  然后找到mkl_cblas.h
export CPATH=$CPATH:/opt/caffe/external/mkl/mklml_lnx_2019.0.20180710/include/

也可直接安装源码（到官网安装）

(1)export CPATH=$CPATH:/opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/include
(2)在cmakelist.txt中添加
include_directories("/opt/intel/compilers_and_libraries_2020.1.217/linux/mkl/include")

cblas.h的问题解决了

错误3：fatal error: caffe2/utils/math/broadcast.h: No such file or directory
解决思路就是去报错的路径中查看是否有相关的文件，发现报错的文件确实不存在，解决思路就是需要把相关的文件添加到路径当中–我再次把pytorch的源代码下载了下来，源代码里面有一个caffe2模块:

1. 下载pytorch源码，找到pytorch里面的caffe2里面的utils然后把里面的额math文件复制到（虚拟境境中的caffe2）  /home/xwt/.conda/envs/densepose/lib/python2.7/site-packages/torch/include/caffe2/utils
2. 
3. （1）cd  /home/xwt/.conda/envs/densepose/lib/python2.7/site-packages/torch/include/caffe2/utils
4. （2）cp  -r   /home/xwt/下载/pytorch-master/caffe2/utils/math/  ./
5. 成功，
6. 新bug出现：fatal error: caffe2/utils/threadpool/ThreadPool.h: 没有那个文件或目录
7. 和上面的解决思路一样的，也是去源码中复制，
8. （1）cd  /home/xwt/.conda/envs/densepose/lib/python2.7/site-packages/torch/include/caffe2/utils
9. （2）cp  -r   /home/xwt/下载/pytorch-master/caffe2/utils/threadpool/    ./
10. 最后运行 make ops。

错误4：error This file was generated by an older version of protoc which is error incompatible with your Protocol Buffer headers.

这是protobuf版本与代码要求版本不一致导致的，确定代码所需要版本为3.6.1。使用conda安装指定版本的protobuf。

conda install protobuf==3.6.1

再次make ops。

验证densepose：

python   ./detectron/tests/test_zero_even_op.py

错误1：OSError: /root/cwt1/DensePose/build/libcaffe2_detectron_custom_ops_gpu.so: undefined symbol: _ZN6caffe219CPUOperatorRegistryB5cxx11Ev
解决方法：将gcc版本从降到4.9.x即可

sudo apt-get install gcc-4.9 g++-4.9
sudo cp /usr/bin/gcc-4.9  /usr/bin/gcc
sudo cp /usr/bin/g++-4.9  /usr/bin/g++

使用图片测试densepose:

python2 tools/infer_simple.py \
    --cfg configs/DensePose_ResNet101_FPN_s1x-e2e.yaml \
    --output-dir DensePoseData/infer_out/ \
    --image-ext jpg \
    --wts https://dl.fbaipublicfiles.com/densepose/DensePose_ResNet101_FPN_s1x-e2e.pkl \
    DensePoseData/demo_data/demo_im.jpg

走到这里就成功了，一步一个坑，鼓励一下自己！