diff --git a/CMakeLists.txt b/CMakeLists.txt
index 0c2e36b..1677f83 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -4,18 +4,18 @@ project(llama_cpp)
 
 option(BUILD_LLAMA_CPP "Build llama.cpp shared library and install alongside python package" ON)
 
-if (APPLE)
-    set(LLAMA_AVX ON CACHE BOOL)
-    set(LLAMA_AVX2 OFF CACHE BOOL)
-    set(LLAMA_FMA OFF CACHE BOOL)
-    set(LLAMA_F16C OFF CACHE BOOL)
-    set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=native -mtune=native")
-    set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -mtune=native")
-endif()
 
 if (BUILD_LLAMA_CPP)
     set(BUILD_SHARED_LIBS "On")
     add_subdirectory(vendor/llama.cpp)
+    if (APPLE)
+        set(LLAMA_AVX OFF)
+        set(LLAMA_AVX2 OFF)
+        set(LLAMA_FMA OFF)
+        set(LLAMA_F16C OFF)
+        set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=native -mtune=native")
+        set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -mtune=native")
+    endif()
     install(
         TARGETS llama 
         LIBRARY DESTINATION ${SKBUILD_PLATLIB_DIR}/llama_cpp