diff --git a/modules/dnn/CMakeLists.txt b/modules/dnn/CMakeLists.txt
index 4d2b959a6..62ec379bd 100644
--- a/modules/dnn/CMakeLists.txt
+++ b/modules/dnn/CMakeLists.txt
@@ -9,9 +9,11 @@ if(${PROTOBUF_FOUND} AND EXISTS ${PROTOBUF_PROTOC_EXECUTABLE})
   file(GLOB PROTO_FILES src/*.proto)
   PROTOBUF_GENERATE_CPP(PROTO_HDRS PROTO_SRCS ${PROTO_FILES})
   set(HAVE_PROTOBUF ON)
+  add_definitions(-DHAVE_PROTOBUF 1)
 else()
   message(STATUS "PROTOBUF not found. Caffe import function will be disabled.")
   set(HAVE_PROTOBUF OFF)
+  set(PROTOBUF_LIBRARIES "")
 endif()
 
 endmacro(_dnn_find_protobuf)
@@ -23,15 +25,8 @@ set(OPENCV_MODULE_IS_PART_OF_WORLD OFF)
 
 ocv_add_module(dnn opencv_imgproc opencv_core opencv_highgui WRAP python matlab)
 ocv_warnings_disable(CMAKE_CXX_FLAGS -Wno-shadow -Wno-parentheses)
-if(HAVE_PROTOBUF)
-add_definitions(-DHAVE_PROTOBUF 1)
 ocv_glob_module_sources(${PROTO_SRCS} ${PROTO_HDRS})
 ocv_source_group("Src\\protobuf" FILES ${PROTO_SRCS} ${PROTO_HDRS})
-else()
-file(GLOB NON_CAFFE_SOURCES "src/*.cpp" "src/layers/*.cpp")
-file(GLOB NON_CAFFE_HEADERS "src/*.hpp" "src/layers/*.hpp")
-ocv_set_module_sources(HEADERS ${NON_CAFFE_HEADERS} SOURCES ${NON_CAFFE_SOURCES})
-endif()
 ocv_module_include_directories(include src/caffe ${PROTOBUF_INCLUDE_DIR})
 
 ocv_create_module(${PROTOBUF_LIBRARIES})
diff --git a/modules/dnn/src/caffe/io.cpp b/modules/dnn/src/caffe/io.cpp
index 7c7e70b63..c4832aac6 100644
--- a/modules/dnn/src/caffe/io.cpp
+++ b/modules/dnn/src/caffe/io.cpp
@@ -1,3 +1,4 @@
+#ifdef HAVE_PROTOBUF
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <google/protobuf/text_format.h>
@@ -65,3 +66,4 @@ bool ReadProtoFromBinaryFile(const char* filename, Message* proto) {
 //}
 
 }  // namespace caffe
+#endif
\ No newline at end of file
diff --git a/modules/dnn/src/caffe/upgrade_proto.cpp b/modules/dnn/src/caffe/upgrade_proto.cpp
index 32ba9020f..388f6d6eb 100644
--- a/modules/dnn/src/caffe/upgrade_proto.cpp
+++ b/modules/dnn/src/caffe/upgrade_proto.cpp
@@ -1,3 +1,4 @@
+#ifdef HAVE_PROTOBUF
 #include <google/protobuf/io/coded_stream.h>
 #include <google/protobuf/io/zero_copy_stream_impl.h>
 #include <google/protobuf/text_format.h>
@@ -938,3 +939,4 @@ void ReadNetParamsFromBinaryFileOrDie(const string& param_file,
 }
 
 }  // namespace caffe
+#endif
\ No newline at end of file