Ubuntu Caffe 설치기

Ubuntu 16.04/Anaconda3/python3.6/opencv 2.4.9.1/CUDA 8.0/CUDNN 5.1/Geforce GTX 1080ti

 

Anaconda로 설치해버리기

--> 걍 한번에 anaconda 환경으로 만들어주는 케이스도 있음

: conda create -n caffe_gpu -c defaults python=3.6 caffe-gpu

 

도움을 주신 사이트

1) 에러가 많이 발생했을 때

https://m.blog.naver.com/PostView.nhn?blogId=ymkim1959&logNo=220851697161&proxyReferer=https%3A%2F%2Fwww.google.com%2F

#1 PC에서 deep learning을 돌려보자.(Caffe/Ubuntu 설치)

 

2) 기본 설치 방법

https://kyubot.tistory.com/93

Ubuntu 16.04 에 Caffe 설치하기(GPU with CUDA)

 

3) 기본 설치 방법(2)

https://www.whydsp.org/337

우분투에서 Caffe 설치하기 매뉴얼

 

sudo apt-get install --no-install-recommends libboost-all-dev

 

https://www.whydsp.org/337

우분투에서 Caffe 설치하기 매뉴얼

ERROR : /usr/include/boost/python/detail/wrap_python.hpp:50:23: fatal error: pyconfig.h: No such file or directory

https://github.com/BVLC/caffe/issues/410

Error 'fatal error: pyconfig.h: No such file or directory' when executing 'make pycaffe' on dev-branch · Issue #410 · BVLC/caffe

 

make all 과정 중 caffe/net.cpp에서 오류가 뜰 때 

src/caffe/net.cpp: In member function ‘void caffe::Net<Dtype>::CopyTrainedLayersFrom(const string&)’:
src/caffe/net.cpp:774:42: error: there are no arguments to ‘H5Fis_hdf5’ that depend on a template parameter, so a declaration of ‘H5Fis_hdf5’ must be available [-fpermissive]
   if (H5Fis_hdf5(trained_filename.c_str())) {
                                          ^
src/caffe/net.cpp:774:42: note: (if you use ‘-fpermissive’, G++ will accept your code, but allowing the use of an undeclared name is deprecated)
src/caffe/net.cpp: In instantiation of ‘void caffe::Net<Dtype>::CopyTrainedLayersFrom(const string&) [with Dtype = float; std::__cxx11::string = std::__cxx11::basic_string<char>]’:
src/caffe/net.cpp:992:1:   required from here
src/caffe/net.cpp:774:17: error: ‘H5Fis_hdf5’ was not declared in this scope
   if (H5Fis_hdf5(trained_filename.c_str())) {
                 ^
src/caffe/net.cpp: In instantiation of ‘void caffe::Net<Dtype>::CopyTrainedLayersFrom(const string&) [with Dtype = double; std::__cxx11::string = std::__cxx11::basic_string<char>]’:
src/caffe/net.cpp:992:1:   required from here
src/caffe/net.cpp:774:17: error: ‘H5Fis_hdf5’ was not declared in this scope
Makefile:591: recipe for target '.build_release/src/caffe/net.o' failed
make: *** [.build_release/src/caffe/net.o] Error 1

https://github.com/BVLC/caffe/issues/6550

Pull request #6346 does not compile - even if verified · Issue #6550 · BVLC/caffe

https://m.blog.naver.com/PostView.nhn?blogId=mingu216&logNo=221292114744&proxyReferer=https%3A%2F%2Fwww.google.com%2F

불러오는 중입니다...

---

20.02.25 [재설치기]

 

Ubuntu16.04 / anaconda3 / python2.7 / cuda8.0 /opencv 2.4.9.1

http://caffe.berkeleyvision.org/install_apt.html

Caffe | Installation: Ubuntu

기본 caffe 설치 방식을 따라 가기.

[설치 시 필요]

$ sudo apt-get install libprotobuf-dev libleveldb-dev libsnappy-dev libopencv-dev libhdf5-serial-dev protobuf-compiler
$ sudo apt-get install --no-install-recommends libboost-all-dev

[CMakeLists.txt]

cmake_minimum_required(VERSION 2.8.7)
if(POLICY CMP0046)
  cmake_policy(SET CMP0046 NEW)
endif()
if(POLICY CMP0054)
  cmake_policy(SET CMP0054 NEW)
endif()

# ---[ Caffe project
project(Caffe C CXX)

# ---[ Caffe version
set(CAFFE_TARGET_VERSION "1.0.0-rc3" CACHE STRING "Caffe logical version")
set(CAFFE_TARGET_SOVERSION "1.0.0-rc3" CACHE STRING "Caffe soname version")
add_definitions(-DCAFFE_VERSION=${CAFFE_TARGET_VERSION})

# ---[ Using cmake scripts and modules
list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/cmake/Modules)

include(ExternalProject)

include(cmake/Utils.cmake)
include(cmake/Targets.cmake)
include(cmake/Misc.cmake)
include(cmake/Summary.cmake)
include(cmake/ConfigGen.cmake)

# ---[ Options
caffe_option(CPU_ONLY  "Build Caffe without CUDA support" OFF) # TODO: rename to USE_CUDA
caffe_option(USE_CUDNN "Build Caffe with cuDNN library support" ON IF NOT CPU_ONLY)
caffe_option(USE_NCCL "Build Caffe with NCCL library support" OFF)
caffe_option(BUILD_SHARED_LIBS "Build shared libraries" ON)
caffe_option(BUILD_python "Build Python wrapper" ON)
set(python_version "2" CACHE STRING "Specify which Python version to use")
caffe_option(BUILD_matlab "Build Matlab wrapper" OFF IF UNIX OR APPLE)
caffe_option(BUILD_docs   "Build documentation" ON IF UNIX OR APPLE)
caffe_option(BUILD_python_layer "Build the Caffe Python layer" ON)
caffe_option(USE_OPENCV "Build with OpenCV support" ON)
caffe_option(USE_LEVELDB "Build with levelDB" ON)
caffe_option(USE_LMDB "Build with lmdb" ON)
caffe_option(ALLOW_LMDB_NOLOCK "Allow MDB_NOLOCK when reading LMDB files (only if necessary)" OFF)

# ---[ Dependencies
include(cmake/Dependencies.cmake)

# ---[ Flags
if(UNIX OR APPLE)
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fPIC -Wall")
endif()

caffe_set_caffe_link()

if(USE_libstdcpp)
  set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -stdlib=libstdc++")
  message("-- Warning: forcing libstdc++ (controlled by USE_libstdcpp option in cmake)")
endif()

add_definitions(-DGTEST_USE_OWN_TR1_TUPLE)

# ---[ Warnings
caffe_warnings_disable(CMAKE_CXX_FLAGS -Wno-sign-compare -Wno-uninitialized)

# ---[ Config generation
configure_file(cmake/Templates/caffe_config.h.in "${PROJECT_BINARY_DIR}/caffe_config.h")

# ---[ Includes
set(Caffe_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/include)
include_directories(${Caffe_INCLUDE_DIR} ${PROJECT_BINARY_DIR})
include_directories(BEFORE src) # This is needed for gtest.

# ---[ Subdirectories
add_subdirectory(src/gtest)
add_subdirectory(src/caffe)
add_subdirectory(tools)
add_subdirectory(examples)
add_subdirectory(python)
add_subdirectory(matlab)
add_subdirectory(docs)

# ---[ Linter target
add_custom_target(lint COMMAND ${CMAKE_COMMAND} -P ${PROJECT_SOURCE_DIR}/cmake/lint.cmake)

# ---[ pytest target
if(BUILD_python)
  add_custom_target(pytest COMMAND python${python_version} -m unittest discover -s caffe/test WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}/python )
  add_dependencies(pytest pycaffe)
endif()

# ---[ Configuration summary
caffe_print_configuration_summary()

# ---[ Export configs generation
caffe_generate_export_configs()

[Makefile.config]

## Refer to http://caffe.berkeleyvision.org/installation.html
# Contributions simplifying and improving our build system are welcome!

# cuDNN acceleration switch (uncomment to build with cuDNN).
USE_CUDNN := 1

# CPU-only switch (uncomment to build without GPU support).
# CPU_ONLY := 1

# uncomment to disable IO dependencies and corresponding data layers
# USE_OPENCV := 0
# USE_LEVELDB := 0
# USE_LMDB := 0

# uncomment to allow MDB_NOLOCK when reading LMDB files (only if necessary)
#	You should not set this flag if you will be reading LMDBs with any
#	possibility of simultaneous read and write
# ALLOW_LMDB_NOLOCK := 1

# Uncomment if you're using OpenCV 3
# OPENCV_VERSION := 3

# To customize your choice of compiler, uncomment and set the following.
# N.B. the default for Linux is g++ and the default for OSX is clang++
# CUSTOM_CXX := g++

# CUDA directory contains bin/ and lib/ directories that we need.
CUDA_DIR := /usr/local/cuda-8.0
# On Ubuntu 14.04, if cuda tools are installed via
# "sudo apt-get install nvidia-cuda-toolkit" then use this instead:
# CUDA_DIR := /usr

# CUDA architecture setting: going with all of them.
# For CUDA < 6.0, comment the *_50 lines for compatibility.
CUDA_ARCH := -gencode arch=compute_20,code=sm_20 \
		-gencode arch=compute_20,code=sm_21 \
		-gencode arch=compute_30,code=sm_30 \
		-gencode arch=compute_35,code=sm_35 \
		-gencode arch=compute_50,code=sm_50 \
		-gencode arch=compute_50,code=compute_50

# BLAS choice:
# atlas for ATLAS (default)
# mkl for MKL
# open for OpenBlas
BLAS := atlas
# Custom (MKL/ATLAS/OpenBLAS) include and lib directories.
# Leave commented to accept the defaults for your choice of BLAS
# (which should work)!
# BLAS_INCLUDE := /path/to/your/blas
# BLAS_LIB := /path/to/your/blas

# Homebrew puts openblas in a directory that is not on the standard search path
# BLAS_INCLUDE := $(shell brew --prefix openblas)/include
# BLAS_LIB := $(shell brew --prefix openblas)/lib

# This is required only if you will compile the matlab interface.
# MATLAB directory should contain the mex binary in /bin.
# MATLAB_DIR := /usr/local
# MATLAB_DIR := /Applications/MATLAB_R2012b.app

# NOTE: this is required only if you will compile the python interface.
# We need to be able to find Python.h and numpy/arrayobject.h.
#PYTHON_INCLUDE := /usr/include/python2.7 \
#		/usr/lib/python2.7/dist-packages/numpy/core/include
# Anaconda Python distribution is quite popular. Include path:
# Verify anaconda location, sometimes it's in root.
ANACONDA_HOME := $(HOME)/anaconda3/envs/caffe_p2
PYTHON_INCLUDE := $(ANACONDA_HOME)/include \
		$(ANACONDA_HOME)/include/python2.7 \
		$(ANACONDA_HOME)/lib/python2.7/site-packages/numpy/core/include

# Uncomment to use Python 3 (default is Python 2)
# PYTHON_LIBRARIES := boost_python3 python3.5m
# PYTHON_INCLUDE := /usr/include/python3.5m \
#                 /usr/lib/python3.5/dist-packages/numpy/core/include

# We need to be able to find libpythonX.X.so or .dylib.
#PYTHON_LIB := /usr/lib
PYTHON_LIB := $(ANACONDA_HOME)/lib

# Homebrew installs numpy in a non standard path (keg only)
# PYTHON_INCLUDE += $(dir $(shell python -c 'import numpy.core; print(numpy.core.__file__)'))/include
# PYTHON_LIB += $(shell brew --prefix numpy)/lib

# Uncomment to support layers written in Python (will link against Python libs)
WITH_PYTHON_LAYER := 1

# Whatever else you find you need goes here.
INCLUDE_DIRS := $(PYTHON_INCLUDE) /usr/local/include /usr/include/hdf5/serial
LIBRARY_DIRS := $(PYTHON_LIB) /usr/local/lib /usr/lib /usr/lib/x86_64-linux-gnu /usr/lib/x86_64-linux-gnu/hdf5/serial

# If Homebrew is installed at a non standard location (for example your home directory) and you use it for general dependencies
# INCLUDE_DIRS += $(shell brew --prefix)/include
# LIBRARY_DIRS += $(shell brew --prefix)/lib

# NCCL acceleration switch (uncomment to build with NCCL)
# https://github.com/NVIDIA/nccl (last tested version: v1.2.3-1+cuda8.0)
# USE_NCCL := 1

# Uncomment to use `pkg-config` to specify OpenCV library paths.
# (Usually not necessary -- OpenCV libraries are normally installed in one of the above $LIBRARY_DIRS.)
# USE_PKG_CONFIG := 1

# N.B. both build and distribute dirs are cleared on `make clean`
BUILD_DIR := build
DISTRIBUTE_DIR := distribute

# Uncomment for debugging. Does not work on OSX due to https://github.com/BVLC/caffe/issues/171
# DEBUG := 1

# The ID of the GPU that 'make runtest' will use to run unit tests.
TEST_GPUID := 0

# enable pretty build (comment to see full commands)
Q ?= @

$ make all(make)
$ make test
$ make runtest

Make Runtest 중 오류

make runtest 과정 중, 오류 발생

.build_release/tools/caffe: error while loading shared libraries: libhdf5_hl.so.100: cannot open shared object file: No such file or directory
Makefile:532: recipe for target 'runtest' failed
make: *** [runtest] Error 127

 

---

해결방법 : [출처 : https://github.com/BVLC/caffe/issues/1463]

gedit ~/.bashrc에서 아래 환경 변수 추가 하거나

export LD_LIBRARY_PATH="/usr/local/cuda/lib64"
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/home/[USER_NAME]/anaconda/lib"

 

아예 파일을 복사해서 넣거나

$ sudo cp ~/anaconda3/lib/libhdf5* /usr/lib/x86_64-linux-gnu/

 

만약 /usr/lib/x86_64-linux-gnu/안에 libhdf5_hl.so.101처럼 다른 버전이 있을 경우,

$ sudo cp libhdf5_hl.so.101 libhdf5_hl.so.100로 변경해도 됨

 

 

make all 중 오류

python3를 쓰는 상황, 앞에서 보이듯 $PYTHON_INCLUDE 변수가 /usr/include/python3.5m인 상황에서 기존의 /usr/lib/x86_64-linux-gnu안에 있는 link파일(libboost_python.so)가 python2로 링킹되어있음

이거를 python3로 바꿔줘야 함

 

$ cd /usr/lib/x86_64-linux-gnu

- 기존의 .so와 .a를 해제해줘야 함

$ sudo unlink libboost_python.a 

$ sudo unlink libboost_python.so

 

- 다시 link 파일을 python3.5m으로 변경해줘야함

$ sudo ln -s libboost_python-py35.so libboost_python3.so

$ sudo ln -s libboost_python-py35.a libboost_python.a

$ sudo ln -s libboost_python-py35.so libboost_python.so

 

혹시 몰라서 bashrc

export CUDA_HOME=/usr/local/cuda-8.0
export PATH=/usr/local/cuda-8.0/bin${PATH:+:${PATH}}
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda-8.0/lib64
export LD_LIBRARY_PATH=/usr/lib/
#export LD_LIBRARY_PATH=/usr/local/cuda-8.0/lib64:$LD_LIBRARY_PATH
export PATH=/usr/local/cuda-8.0/bin:$PATH
export LD_LIBRARY_PATH=/usr/local/cuda-8.0/lib64${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
#export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/home/ai8503/anaconda3/lib"
export MATTERPORT_DATA_DIR=/media/ai8503/f4837b59-5cdd-4faa-955b-befb9a99e668/JS/v1/scans
# for caffe
export PYTHONPATH=/media/ai8503/f4837b59-5cdd-4faa-955b-befb9a99e668/JS/bottom-up-attention/caffe/python:$PYTHONPATH