From 9201de2b49eafd4872f9cf8c43d50fcc9d3d8c28 Mon Sep 17 00:00:00 2001 From: T <3923106166@qq.com> Date: Mon, 13 Jan 2025 11:39:17 +0800 Subject: [PATCH] [swift] add module omnivlm (#41) * support omnivlm for ios * add swift test cases * update test case --- .gitignore | 3 +- Package.swift | 202 +++++++++++++++++++----------- examples/omni-vlm/build-info.cpp | 5 + spm/omnivlm/omni-vlm-wrapper.h | 1 + swift/LlavaTests/LlavaTests.swift | 27 ++++ 5 files changed, 164 insertions(+), 74 deletions(-) create mode 100644 examples/omni-vlm/build-info.cpp create mode 120000 spm/omnivlm/omni-vlm-wrapper.h create mode 100644 swift/LlavaTests/LlavaTests.swift diff --git a/.gitignore b/.gitignore index e390b124a..da01a6b42 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,2 @@ -build/** \ No newline at end of file +build/** +.build/** \ No newline at end of file diff --git a/Package.swift b/Package.swift index 6b68aecde..be98b19bd 100644 --- a/Package.swift +++ b/Package.swift @@ -3,87 +3,143 @@ import PackageDescription var sources = [ - "src/llama.cpp", - "src/llama-vocab.cpp", - "src/llama-grammar.cpp", - "src/llama-sampling.cpp", - "src/unicode.cpp", - "src/unicode-data.cpp", - "ggml/src/ggml.c", - "ggml/src/ggml-aarch64.c", - "ggml/src/ggml-alloc.c", - "ggml/src/ggml-backend.cpp", - "ggml/src/ggml-backend-reg.cpp", - "ggml/src/ggml-cpu/ggml-cpu.c", - "ggml/src/ggml-cpu/ggml-cpu.cpp", - "ggml/src/ggml-cpu/ggml-cpu-aarch64.c", - "ggml/src/ggml-cpu/ggml-cpu-quants.c", - "ggml/src/ggml-threading.cpp", - "ggml/src/ggml-quants.c", + "src/llama.cpp", + "src/llama-vocab.cpp", + "src/llama-grammar.cpp", + "src/llama-sampling.cpp", + "src/unicode.cpp", + "src/unicode-data.cpp", + "ggml/src/ggml.c", + "ggml/src/ggml-aarch64.c", + "ggml/src/ggml-alloc.c", + "ggml/src/ggml-backend.cpp", + "ggml/src/ggml-backend-reg.cpp", + "ggml/src/ggml-cpu/ggml-cpu.c", + "ggml/src/ggml-cpu/ggml-cpu.cpp", + "ggml/src/ggml-cpu/ggml-cpu-aarch64.c", + "ggml/src/ggml-cpu/ggml-cpu-quants.c", + "ggml/src/ggml-threading.cpp", + "ggml/src/ggml-quants.c", +] + +var omniVlmSources = [ + "common/log.h", + "common/log.cpp", + "common/arg.h", + "common/arg.cpp", + "common/common.cpp", + "common/common.h", + "common/json.hpp", + "common/json-schema-to-grammar.cpp", + "common/json-schema-to-grammar.h", + "src/llama-grammar.h", + "common/grammar-parser.cpp", + "common/grammar-parser.h", + "common/sampling.cpp", + "common/sampling.h", + "examples/omni-vlm/build-info.cpp", + "examples/omni-vlm/clip.cpp", + "examples/omni-vlm/clip.h", + "examples/omni-vlm/omni-vlm-wrapper.cpp", + "examples/omni-vlm/omni-vlm-wrapper.h", + "examples/omni-vlm/omni-vlm.h", + "examples/omni-vlm/omni-vlm.cpp", + "common/base64.cpp", + "ggml/include/ggml.h", + "ggml/include/ggml-alloc.h", + "ggml/include/ggml-backend.h", + "ggml/src/ggml-common.h", +] + +var testSources = [ + "swift/LlavaTests/LlavaTests.swift" ] var resources: [Resource] = [] var linkerSettings: [LinkerSetting] = [] -var cSettings: [CSetting] = [ - .unsafeFlags(["-Wno-shorten-64-to-32", "-O3", "-DNDEBUG"]), - .unsafeFlags(["-fno-objc-arc"]), - .headerSearchPath("ggml/src"), - // NOTE: NEW_LAPACK will required iOS version 16.4+ - // We should consider add this in the future when we drop support for iOS 14 - // (ref: ref: https://developer.apple.com/documentation/accelerate/1513264-cblas_sgemm?language=objc) - // .define("ACCELERATE_NEW_LAPACK"), - // .define("ACCELERATE_LAPACK_ILP64") +var cSettings: [CSetting] = [ + .unsafeFlags(["-Wno-shorten-64-to-32", "-O3", "-DNDEBUG"]), + .unsafeFlags(["-fno-objc-arc"]), + .headerSearchPath("."), + .headerSearchPath("ggml/src"), + .headerSearchPath("common"), + .unsafeFlags(["-framework", "Foundation"]), + .unsafeFlags(["-framework", "Accelerate"]), ] -#if canImport(Darwin) -sources.append("ggml/src/ggml-common.h") -sources.append("ggml/src/ggml-metal/ggml-metal.m") -resources.append(.process("ggml/src/ggml-metal/ggml-metal.metal")) -linkerSettings.append(.linkedFramework("Accelerate")) -cSettings.append( - contentsOf: [ - .define("GGML_USE_ACCELERATE"), - .define("GGML_USE_METAL") - ] -) +#if os(Linux) + cSettings.append(.define("_GNU_SOURCE")) #endif -#if os(Linux) - cSettings.append(.define("_GNU_SOURCE")) -#endif +let baseSettings = cSettings + [ + .headerSearchPath("."), + .headerSearchPath("src"), + .headerSearchPath("common"), + .headerSearchPath("examples/omni-vlm"), + .headerSearchPath("ggml/include"), +] + +let llamaTarget = Target.target( + name: "llama", + dependencies: [], + path: ".", + exclude: [ + "build", "cmake", "examples", "scripts", "models", + "tests", "CMakeLists.txt", "Makefile", + ], + sources: sources, + resources: resources, + publicHeadersPath: "spm-headers", + cSettings: cSettings, + linkerSettings: linkerSettings +) + +let omnivlmTarget = Target.target( + name: "omnivlm", + dependencies: ["llama"], + path: ".", + sources: omniVlmSources, + resources: resources, + publicHeadersPath: "spm/omnivlm", + cSettings: baseSettings + [ + .headerSearchPath("ggml/src"), + ], + cxxSettings: [.unsafeFlags(["-std=c++14"])], + linkerSettings: linkerSettings +) + +let testTarget = Target.testTarget( + name: "LlavaTests", + dependencies: ["omnivlm"], + path: ".", + sources: testSources, + resources: resources, + cSettings: baseSettings + [ + .headerSearchPath("ggml/src"), + ], + linkerSettings: linkerSettings +) + +let supportedPlatforms: [SupportedPlatform] = [ + .macOS(.v12), + .iOS(.v14), + .watchOS(.v4), + .tvOS(.v14) +] + +let products = [ + Product.library(name: "llama", targets: ["llama"]), + Product.library(name: "omnivlm", targets: ["omnivlm"]) +] let package = Package( - name: "llama", - platforms: [ - .macOS(.v12), - .iOS(.v14), - .watchOS(.v4), - .tvOS(.v14) - ], - products: [ - .library(name: "llama", targets: ["llama"]), - ], - targets: [ - .target( - name: "llama", - path: ".", - exclude: [ - "build", - "cmake", - "examples", - "scripts", - "models", - "tests", - "CMakeLists.txt", - "Makefile", - "ggml/src/ggml-metal-embed.metal" - ], - sources: sources, - resources: resources, - publicHeadersPath: "spm-headers", - cSettings: cSettings, - linkerSettings: linkerSettings - ) - ], - cxxLanguageStandard: .cxx11 + name: "llama", + platforms: supportedPlatforms, + products: products, + targets: [ + llamaTarget, + omnivlmTarget, + testTarget + ], + cxxLanguageStandard: .cxx14 ) diff --git a/examples/omni-vlm/build-info.cpp b/examples/omni-vlm/build-info.cpp new file mode 100644 index 000000000..2df883f4d --- /dev/null +++ b/examples/omni-vlm/build-info.cpp @@ -0,0 +1,5 @@ +// src/build-info.cpp +int LLAMA_BUILD_NUMBER = 0; +const char* LLAMA_BUILD_TARGET = "ios-arm64"; +const char* LLAMA_COMMIT = "unknown"; +const char* LLAMA_COMPILER = "clang"; \ No newline at end of file diff --git a/spm/omnivlm/omni-vlm-wrapper.h b/spm/omnivlm/omni-vlm-wrapper.h new file mode 120000 index 000000000..0ed29bb49 --- /dev/null +++ b/spm/omnivlm/omni-vlm-wrapper.h @@ -0,0 +1 @@ +../../examples/omni-vlm/omni-vlm-wrapper.h \ No newline at end of file diff --git a/swift/LlavaTests/LlavaTests.swift b/swift/LlavaTests/LlavaTests.swift new file mode 100644 index 000000000..9d5157389 --- /dev/null +++ b/swift/LlavaTests/LlavaTests.swift @@ -0,0 +1,27 @@ +import XCTest +@testable import omnivlm + +final class LlavaTests: XCTestCase { + func testOmniVlm() { + omnivlm_init("model-q4_0.gguf", + "projector-q4_0.gguf", + "vlm-81-instruct") + + let startTime = Date() + + if let cString = omnivlm_inference("describe the image", "cat.png") { + let res = String(cString: cString) + print("res: \(res)") + + let endTime = Date() + let inferenceTime = endTime.timeIntervalSince(startTime) + print("Inference time: \(inferenceTime) seconds") + + XCTAssertFalse(res.isEmpty, "res should not be null") + } else { + XCTFail("failed") + } + + omnivlm_free() + } +}